From 595b45e67c83ac02f7c3810da6c80c28a95a4feb Mon Sep 17 00:00:00 2001 From: Devashish Date: Tue, 5 Mar 2024 14:43:51 -0500 Subject: [PATCH] Packer tracks Version and Plugins Metadata (#12860) --- .../types.build.hcp_packer_registry_test.go | 3 ++ hcl2template/types.build_test.go | 34 ++++++++++------ hcl2template/types.datasource_test.go | 2 + hcl2template/types.packer_config.go | 1 + hcl2template/types.packer_config_test.go | 10 +++-- hcl2template/types.source_test.go | 1 + hcl2template/types.variables_test.go | 13 ++++-- internal/hcp/registry/hcl.go | 12 ++++++ internal/hcp/registry/json.go | 15 ++++++- packer/build.go | 40 +++++++++++++++++++ packer/plugin.go | 29 ++++++++++++++ 11 files changed, 139 insertions(+), 21 deletions(-) diff --git a/hcl2template/types.build.hcp_packer_registry_test.go b/hcl2template/types.build.hcp_packer_registry_test.go index b27b80e5b..b43566d3e 100644 --- a/hcl2template/types.build.hcp_packer_registry_test.go +++ b/hcl2template/types.build.hcp_packer_registry_test.go @@ -57,6 +57,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, + BuilderType: "virtualbox-iso", }, }, false, @@ -110,6 +111,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, + BuilderType: "virtualbox-iso", }, }, false, @@ -241,6 +243,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, Prepared: true, + BuilderType: "null", }, }, false, diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index 8647821dd..d94dee4f8 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -283,6 +283,7 @@ func TestParse_build(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -317,6 +318,7 @@ func TestParse_build(t *testing.T) { }, &packer.CoreBuild{ Type: "amazon-ebs.aws-ubuntu-16.04", + BuilderType: "amazon-ebs", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -397,9 +399,10 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -427,9 +430,10 @@ func TestParse_build(t *testing.T) { PostProcessors: [][]packer.CoreBuildPostProcessor{}, }, &packer.CoreBuild{ - Type: "amazon-ebs.aws-ubuntu-16.04", - Prepared: true, - Builder: emptyMockBuilder, + Type: "amazon-ebs.aws-ubuntu-16.04", + BuilderType: "amazon-ebs", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "file", @@ -486,9 +490,10 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -550,6 +555,7 @@ func TestParse_build(t *testing.T) { &packer.CoreBuild{ BuildName: "build-name", Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -604,6 +610,7 @@ func TestParse_build(t *testing.T) { &packer.CoreBuild{ BuildName: "test-build", Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -659,10 +666,11 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - BuildName: "build-name-test", - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + BuildName: "build-name-test", + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PName: "build-name-test", diff --git a/hcl2template/types.datasource_test.go b/hcl2template/types.datasource_test.go index d12fcafd0..69f654138 100644 --- a/hcl2template/types.datasource_test.go +++ b/hcl2template/types.datasource_test.go @@ -57,6 +57,7 @@ func TestParse_datasource(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -134,6 +135,7 @@ func TestParse_datasource(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 3ead0b6b5..d21d10ede 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -622,6 +622,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder) pcb.HCLConfig = decoded + pcb.BuilderType = srcUsage.Type // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 69bc2da2c..391899e92 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -208,8 +208,9 @@ func TestParser_complete(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, Builder: &MockBuilder{ Config: MockConfig{ NestedMockConfig: NestedMockConfig{ @@ -319,8 +320,9 @@ func TestParser_complete(t *testing.T) { }, }, &packer.CoreBuild{ - Type: "amazon-ebs.ubuntu-1604", - Prepared: true, + Type: "amazon-ebs.ubuntu-1604", + BuilderType: "amazon-ebs", + Prepared: true, Builder: &MockBuilder{ Config: MockConfig{ NestedMockConfig: NestedMockConfig{ diff --git a/hcl2template/types.source_test.go b/hcl2template/types.source_test.go index 47caeec5a..e31916368 100644 --- a/hcl2template/types.source_test.go +++ b/hcl2template/types.source_test.go @@ -55,6 +55,7 @@ func TestParse_source(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/hcl2template/types.variables_test.go b/hcl2template/types.variables_test.go index a19b80bd9..ecff3677c 100644 --- a/hcl2template/types.variables_test.go +++ b/hcl2template/types.variables_test.go @@ -129,6 +129,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -293,6 +294,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -379,6 +381,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -442,6 +445,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -483,6 +487,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -538,9 +543,10 @@ func TestParse_variables(t *testing.T) { }, false, false, []packersdk.Build{&packer.CoreBuild{ - Type: "null.null-builder", - Prepared: true, - Builder: &null.Builder{}, + Type: "null.null-builder", + BuilderType: "null", + Prepared: true, + Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -627,6 +633,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index 23c5af7cc..ce6fd68ea 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -85,6 +85,18 @@ func (h *HCLRegistry) CompleteBuild( if ok { name = cb.Type } + + metadata := cb.GetMetadata() + log.Printf( + "[TRACE] HCL 'Packer Version' Metadata for build name %q: %q\n", + name, metadata.PackerVersion, + ) + for k, pluginDetails := range metadata.Plugins { + log.Printf( + "[TRACE] HCL 'Plugin' Metadata for build name %q: %q -- %q\n", + name, k, pluginDetails.Description.Version, + ) + } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index b35f2ea49..b86cfd452 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -93,7 +93,20 @@ func (h *JSONRegistry) CompleteBuild( artifacts []sdkpacker.Artifact, buildErr error, ) ([]sdkpacker.Artifact, error) { - return h.bucket.completeBuild(ctx, build.Name(), artifacts, buildErr) + name := build.Name() + + metadata := build.(*packer.CoreBuild).GetMetadata() + log.Printf( + "[TRACE] JSON 'Packer Version' Metadata for build name %q: %q\n", + name, metadata.PackerVersion, + ) + for k, pluginDetails := range metadata.Plugins { + log.Printf( + "[TRACE] JSON 'Plugin' Metadata for build name %q: %q -- %q\n", + name, k, pluginDetails.Description.Version, + ) + } + return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } // VersionStatusSummary prints a status report in the UI if the version is not yet done diff --git a/packer/build.go b/packer/build.go index e51e71d00..9dca9610a 100644 --- a/packer/build.go +++ b/packer/build.go @@ -52,6 +52,46 @@ type CoreBuild struct { prepareCalled bool } +type BuildMetadata struct { + PackerVersion string + Plugins map[string]PluginDetails +} + +func (b *CoreBuild) getPluginsMetadata() map[string]PluginDetails { + resp := map[string]PluginDetails{} + + builderPlugin, builderPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, b.BuilderType)] + if builderPluginOk { + resp[builderPlugin.Name] = builderPlugin + } + + for _, pp := range b.PostProcessors { + for _, p := range pp { + postprocessorsPlugin, postprocessorsPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, p.PType)] + if postprocessorsPluginOk { + resp[postprocessorsPlugin.Name] = postprocessorsPlugin + } + } + } + + for _, pv := range b.Provisioners { + provisionerPlugin, provisionerPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, pv.PType)] + if provisionerPluginOk { + resp[provisionerPlugin.Name] = provisionerPlugin + } + } + + return resp +} + +func (b *CoreBuild) GetMetadata() BuildMetadata { + metadata := BuildMetadata{ + PackerVersion: version.FormattedVersion(), + Plugins: b.getPluginsMetadata(), + } + return metadata +} + // CoreBuildPostProcessor Keeps track of the post-processor and the // configuration of the post-processor used within a build. type CoreBuildPostProcessor struct { diff --git a/packer/plugin.go b/packer/plugin.go index 30261d7e0..513694f63 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -6,6 +6,7 @@ package packer import ( "crypto/sha256" "encoding/json" + "fmt" "log" "os" "os/exec" @@ -135,6 +136,11 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error } pluginPrefix := pluginName + "-" + pluginDetails := PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } for _, builderName := range desc.Builders { builderName := builderName // copy to avoid pointer overwrite issue @@ -145,6 +151,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, key)] = pluginDetails + } if len(desc.Builders) > 0 { @@ -160,6 +168,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key)] = pluginDetails } if len(desc.PostProcessors) > 0 { @@ -175,6 +184,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, key)] = pluginDetails + } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) @@ -189,6 +200,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.DataSources.Set(key, func() (packersdk.Datasource, error) { return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource() }) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentDataSource, key)] = pluginDetails } if len(desc.Datasources) > 0 { log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName) @@ -240,3 +252,20 @@ func (c *PluginConfig) Client(path string, args ...string) *PluginClient { config.MaxPort = c.PluginMaxPort return NewClient(&config) } + +type PluginComponentType string + +const ( + PluginComponentBuilder PluginComponentType = "builder" + PluginComponentPostProcessor PluginComponentType = "post-processor" + PluginComponentProvisioner PluginComponentType = "provisioner" + PluginComponentDataSource PluginComponentType = "data-source" +) + +type PluginDetails struct { + Name string + Description pluginsdk.SetDescription + PluginPath string +} + +var PluginsDetailsStorage = map[string]PluginDetails{}