From bfb9df00398f4a1f2ec96675040c68db8e45dd9a Mon Sep 17 00:00:00 2001 From: Moss Date: Tue, 9 Jun 2020 15:23:29 +0200 Subject: [PATCH 1/3] fix except flag for JSON and HCL2 --- command/build_test.go | 70 ++++++++++++++----- .../test-fixtures/build-only/template.pkr.hcl | 49 +++++++++++++ .../validate/validate_except.json | 26 +++++++ .../validate/validate_except.pkr.hcl | 23 ++++++ command/validate_test.go | 53 ++++++++++++++ hcl2template/types.packer_config.go | 23 ++++++ packer/core.go | 2 + 7 files changed, 227 insertions(+), 19 deletions(-) create mode 100644 command/test-fixtures/build-only/template.pkr.hcl create mode 100644 command/test-fixtures/validate/validate_except.json create mode 100644 command/test-fixtures/validate/validate_except.pkr.hcl diff --git a/command/build_test.go b/command/build_test.go index a1c9a2f71..418f95724 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -368,29 +368,61 @@ func TestBuildExceptFileCommaFlags(t *testing.T) { c := &BuildCommand{ Meta: testMetaFile(t), } - - args := []string{ - "-parallel-builds=1", - "-except=chocolate,vanilla", - filepath.Join(testFixture("build-only"), "template.json"), + tc := []struct { + name string + args []string + expectedFiles []string + buildNotExpectedFiles []string + postProcNotExpectedFiles []string + }{ + { + name: "JSON: except build and post-processor", + args: []string{ + "-parallel-builds=1", + "-except=chocolate,vanilla,pear", + filepath.Join(testFixture("build-only"), "template.json"), + }, + expectedFiles: []string{"apple.txt", "cherry.txt", "peach.txt"}, + buildNotExpectedFiles: []string{"chocolate.txt", "vanilla.txt", "tomato.txt", "unnamed.txt"}, + postProcNotExpectedFiles: []string{"pear.txt"}, + }, + { + name: "HCL2: except build and post-processor", + args: []string{ + "-parallel-builds=1", + "-except=file.chocolate,file.vanilla,pear", + filepath.Join(testFixture("build-only"), "template.pkr.hcl"), + }, + expectedFiles: []string{"apple.txt", "cherry.txt", "peach.txt"}, + buildNotExpectedFiles: []string{"chocolate.txt", "vanilla.txt", "tomato.txt", "unnamed.txt"}, + postProcNotExpectedFiles: []string{"pear.txt"}, + }, } - defer cleanup() + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + defer cleanup() - if code := c.Run(args); code != 0 { - fatalCommand(t, c.Meta) - } + if code := c.Run(tt.args); code != 0 { + fatalCommand(t, c.Meta) + } - for _, f := range []string{"chocolate.txt", "vanilla.txt", "tomato.txt", - "unnamed.txt"} { - if fileExists(f) { - t.Errorf("Expected NOT to find %s", f) - } - } - for _, f := range []string{"apple.txt", "cherry.txt", "pear.txt", "peach.txt"} { - if !fileExists(f) { - t.Errorf("Expected to find %s", f) - } + for _, f := range tt.buildNotExpectedFiles { + if fileExists(f) { + t.Errorf("build not skipped: Expected NOT to find %s", f) + } + } + for _, f := range tt.postProcNotExpectedFiles { + if fileExists(f) { + t.Errorf("post-processor not skipped: Expected NOT to find %s", f) + } + } + for _, f := range tt.expectedFiles { + if !fileExists(f) { + t.Errorf("Expected to find %s", f) + } + } + }) } } diff --git a/command/test-fixtures/build-only/template.pkr.hcl b/command/test-fixtures/build-only/template.pkr.hcl new file mode 100644 index 000000000..12ddef71a --- /dev/null +++ b/command/test-fixtures/build-only/template.pkr.hcl @@ -0,0 +1,49 @@ +source "file" "chocolate" { + content = "chocolate" + target = "chocolate.txt" +} + +source "file" "vanilla" { + content = "vanilla" + target = "vanilla.txt" +} + +source "file" "cherry" { + content = "cherry" + target = "cherry.txt" +} + + +build { + sources = [ + "sources.file.chocolate", + "sources.file.vanilla", + "sources.file.cherry", + ] + + post-processor "shell-local" { + name = "apple" + inline = [ "echo apple > apple.txt" ] + } + + post-processor "shell-local" { + name = "peach" + inline = [ "echo apple > peach.txt" ] + } + + post-processor "shell-local" { + name = "pear" + inline = [ "echo apple > pear.txt" ] + } + + post-processor "shell-local" { + only = ["vanilla"] + name = "tomato" + inline = [ "echo apple > tomato.txt" ] + } + + post-processor "shell-local" { + only = ["chocolate"] + inline = [ "echo apple > unnamed.txt" ] + } +} diff --git a/command/test-fixtures/validate/validate_except.json b/command/test-fixtures/validate/validate_except.json new file mode 100644 index 000000000..0dcbe1446 --- /dev/null +++ b/command/test-fixtures/validate/validate_except.json @@ -0,0 +1,26 @@ +{ + "builders":[ + { + "name": "chocolate", + "type":"file", + "target":"chocolate.txt", + "content":"chocolate" + }, + { + "type":"file", + "name": "vanilla" + } + ], + "post-processors": [ + { + "name": "apple", + "type": "shell-local", + "inline": [ "echo apple 'hello'" ] + }, + { + "name": "pear", + "type": "shell-local" + } + ], + "min_packer_version":"101.0.0" +} diff --git a/command/test-fixtures/validate/validate_except.pkr.hcl b/command/test-fixtures/validate/validate_except.pkr.hcl new file mode 100644 index 000000000..1ffd76dc4 --- /dev/null +++ b/command/test-fixtures/validate/validate_except.pkr.hcl @@ -0,0 +1,23 @@ +source "file" "chocolate" { + content = "chocolate" + target = "chocolate.txt" +} +source "file" "vanilla" { + content = "vanilla" +} + +build { + sources = [ + "source.file.chocolate", + "source.file.vanilla" + ] + + post-processor "shell-local" { + name = "apple" + inline = [ "echo apple 'hello'" ] + } + + post-processor "shell-local" { + name = "pear" + } +} diff --git a/command/validate_test.go b/command/validate_test.go index 67c84f5e9..0c14e3e2b 100644 --- a/command/validate_test.go +++ b/command/validate_test.go @@ -98,3 +98,56 @@ func TestValidateCommandBadVersion(t *testing.T) { } t.Log(stdout) } + +func TestValidateCommandExcept(t *testing.T) { + tt := []struct { + name string + args []string + exitCode int + }{ + { + name: "JSON: validate except build and post-processor", + args: []string{ + "-except=vanilla,pear", + filepath.Join(testFixture("validate"), "validate_except.json"), + }, + }, + { + name: "JSON: fail validate except build and post-processor", + args: []string{ + "-except=chocolate,apple", + filepath.Join(testFixture("validate"), "validate_except.json"), + }, + exitCode: 1, + }, + { + name: "HCL2: validate except build and post-processor", + args: []string{ + "-except=file.vanilla,pear", + filepath.Join(testFixture("validate"), "validate_except.pkr.hcl"), + }, + }, + { + name: "HCL2: fail validation except build and post-processor", + args: []string{ + "-except=file.chocolate,apple", + filepath.Join(testFixture("validate"), "validate_except.pkr.hcl"), + }, + exitCode: 1, + }, + } + + c := &ValidateCommand{ + Meta: testMetaFile(t), + } + c.CoreConfig.Version = "102.0.0" + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + tc := tc + if code := c.Run(tc.args); code != tc.exitCode { + fatalCommand(t, c.Meta) + } + }) + } +} diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index fd5d00a1d..6fdc9ce3d 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -2,6 +2,7 @@ package hcl2template import ( "fmt" + "github.com/gobwas/glob" "strings" "github.com/hashicorp/hcl/v2" @@ -37,6 +38,9 @@ type PackerConfig struct { provisionersSchemas packer.ProvisionerStore postProcessorsSchemas packer.PostProcessorStore + + except []glob.Glob + only []glob.Glob } type ValidationOptions struct { @@ -244,6 +248,23 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks [ if ppb.OnlyExcept.Skip(source.Type + "." + source.Name) { continue } + + name := ppb.PName + if name == "" { + name = ppb.PType + } + // -except + exclude := false + for _, exceptGlob := range cfg.except { + if exceptGlob.Match(name) { + exclude = true + break + } + } + if exclude { + continue + } + postProcessor, moreDiags := cfg.startPostProcessor(source, ppb, ectx, generatedVars) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { @@ -289,6 +310,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build if diags.HasErrors() { return nil, diags } + cfg.only = onlyGlobs include := false for _, onlyGlob := range onlyGlobs { if onlyGlob.Match(buildName) { @@ -307,6 +329,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build if diags.HasErrors() { return nil, diags } + cfg.except = exceptGlobs exclude := false for _, exceptGlob := range exceptGlobs { if exceptGlob.Match(buildName) { diff --git a/packer/core.go b/packer/core.go index 84b8b1bb5..8f60c0b23 100644 --- a/packer/core.go +++ b/packer/core.go @@ -133,6 +133,8 @@ func (c *Core) BuildNames(only, except []string) []string { sort.Strings(only) sort.Strings(except) + c.except = except + c.only = only r := make([]string, 0, len(c.builds)) for n := range c.builds { From 6599e7ab05f5aa754ddbb1d8e77d9b464af499fe Mon Sep 17 00:00:00 2001 From: Moss Date: Tue, 9 Jun 2020 15:29:32 +0200 Subject: [PATCH 2/3] apply goimports --- hcl2template/types.packer_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 6fdc9ce3d..9a999a62b 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -2,9 +2,9 @@ package hcl2template import ( "fmt" - "github.com/gobwas/glob" "strings" + "github.com/gobwas/glob" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/packer/helper/common" From daaeccd8b006362c64bfbd3376a545fe648830df Mon Sep 17 00:00:00 2001 From: Moss Date: Tue, 9 Jun 2020 17:35:53 +0200 Subject: [PATCH 3/3] Skip consecutives post-processors --- command/build_test.go | 9 ++-- .../test-fixtures/build-only/template.json | 5 +++ .../test-fixtures/build-only/template.pkr.hcl | 5 +++ .../validate/validate_except.json | 43 ++++++++++++------- .../validate/validate_except.pkr.hcl | 7 ++- command/validate_test.go | 2 + hcl2template/types.packer_config.go | 2 +- packer/core.go | 2 +- 8 files changed, 53 insertions(+), 22 deletions(-) diff --git a/command/build_test.go b/command/build_test.go index 418f95724..f5e8d5a74 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -379,23 +379,23 @@ func TestBuildExceptFileCommaFlags(t *testing.T) { name: "JSON: except build and post-processor", args: []string{ "-parallel-builds=1", - "-except=chocolate,vanilla,pear", + "-except=chocolate,vanilla,tomato", filepath.Join(testFixture("build-only"), "template.json"), }, expectedFiles: []string{"apple.txt", "cherry.txt", "peach.txt"}, buildNotExpectedFiles: []string{"chocolate.txt", "vanilla.txt", "tomato.txt", "unnamed.txt"}, - postProcNotExpectedFiles: []string{"pear.txt"}, + postProcNotExpectedFiles: []string{"pear.txt, banana.txt"}, }, { name: "HCL2: except build and post-processor", args: []string{ "-parallel-builds=1", - "-except=file.chocolate,file.vanilla,pear", + "-except=file.chocolate,file.vanilla,tomato", filepath.Join(testFixture("build-only"), "template.pkr.hcl"), }, expectedFiles: []string{"apple.txt", "cherry.txt", "peach.txt"}, buildNotExpectedFiles: []string{"chocolate.txt", "vanilla.txt", "tomato.txt", "unnamed.txt"}, - postProcNotExpectedFiles: []string{"pear.txt"}, + postProcNotExpectedFiles: []string{"pear.txt, banana.txt"}, }, } @@ -637,6 +637,7 @@ func cleanup(moreFiles ...string) { os.RemoveAll("cherry.txt") os.RemoveAll("apple.txt") os.RemoveAll("peach.txt") + os.RemoveAll("banana.txt") os.RemoveAll("pear.txt") os.RemoveAll("tomato.txt") os.RemoveAll("unnamed.txt") diff --git a/command/test-fixtures/build-only/template.json b/command/test-fixtures/build-only/template.json index d00f4e516..03982a1e9 100644 --- a/command/test-fixtures/build-only/template.json +++ b/command/test-fixtures/build-only/template.json @@ -37,6 +37,11 @@ "name": "pear", "type": "shell-local", "inline": [ "echo pear > pear.txt" ] + }, + { + "name": "banana", + "type": "shell-local", + "inline": [ "echo pear > banana.txt" ] } ], [ diff --git a/command/test-fixtures/build-only/template.pkr.hcl b/command/test-fixtures/build-only/template.pkr.hcl index 12ddef71a..408579991 100644 --- a/command/test-fixtures/build-only/template.pkr.hcl +++ b/command/test-fixtures/build-only/template.pkr.hcl @@ -36,6 +36,11 @@ build { inline = [ "echo apple > pear.txt" ] } + post-processor "shell-local" { + name = "banana" + inline = [ "echo apple > banana.txt" ] + } + post-processor "shell-local" { only = ["vanilla"] name = "tomato" diff --git a/command/test-fixtures/validate/validate_except.json b/command/test-fixtures/validate/validate_except.json index 0dcbe1446..74e1f3f98 100644 --- a/command/test-fixtures/validate/validate_except.json +++ b/command/test-fixtures/validate/validate_except.json @@ -1,26 +1,39 @@ { - "builders":[ + "builders": [ { "name": "chocolate", - "type":"file", - "target":"chocolate.txt", - "content":"chocolate" + "type": "file", + "target": "chocolate.txt", + "content": "chocolate" }, { - "type":"file", + "type": "file", "name": "vanilla" } ], "post-processors": [ - { - "name": "apple", - "type": "shell-local", - "inline": [ "echo apple 'hello'" ] - }, - { - "name": "pear", - "type": "shell-local" - } + [ + { + "name": "apple", + "type": "shell-local", + "inline": [ + "echo apple 'apple'" + ] + } + ], + [ + { + "name": "pear", + "type": "shell-local", + "inline": [ + "echo apple 'pear'" + ] + }, + { + "name": "banana", + "type": "shell-local" + } + ] ], - "min_packer_version":"101.0.0" + "min_packer_version": "101.0.0" } diff --git a/command/test-fixtures/validate/validate_except.pkr.hcl b/command/test-fixtures/validate/validate_except.pkr.hcl index 1ffd76dc4..4be7e35c9 100644 --- a/command/test-fixtures/validate/validate_except.pkr.hcl +++ b/command/test-fixtures/validate/validate_except.pkr.hcl @@ -14,10 +14,15 @@ build { post-processor "shell-local" { name = "apple" - inline = [ "echo apple 'hello'" ] + inline = [ "echo apple 'apple'" ] } post-processor "shell-local" { name = "pear" + inline = [ "echo apple 'pear'" ] + } + + post-processor "shell-local" { + name = "banana" } } diff --git a/command/validate_test.go b/command/validate_test.go index 0c14e3e2b..5dfbd97a9 100644 --- a/command/validate_test.go +++ b/command/validate_test.go @@ -144,6 +144,8 @@ func TestValidateCommandExcept(t *testing.T) { for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { + defer cleanup() + tc := tc if code := c.Run(tc.args); code != tc.exitCode { fatalCommand(t, c.Meta) diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 9a999a62b..5f1604958 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -262,7 +262,7 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks [ } } if exclude { - continue + break } postProcessor, moreDiags := cfg.startPostProcessor(source, ppb, ectx, generatedVars) diff --git a/packer/core.go b/packer/core.go index 8f60c0b23..1198cc341 100644 --- a/packer/core.go +++ b/packer/core.go @@ -321,7 +321,7 @@ func (c *Core) Build(n string) (Build, error) { } } if foundExcept { - continue + break } // Get the post-processor