From 97070432f2dfdbfd50bb58e2e6a0fd5170b63b0c Mon Sep 17 00:00:00 2001 From: sroomberg Date: Sun, 19 Apr 2026 12:12:40 -0400 Subject: [PATCH] issue 13537: add tag-based filtering --- command/build.go | 9 + command/build_test.go | 58 ++++ command/cli.go | 9 +- .../test-fixtures/hcl-filter/build.pkr.hcl | 38 +++ command/validate.go | 5 +- hcl2template/plugin.go | 8 +- hcl2template/types.build.go | 31 ++ hcl2template/types.packer_config.go | 83 +++++ hcl2template/types.source.go | 123 +++++++- packer/build.go | 18 ++ packer/buildfilter/filter.go | 288 ++++++++++++++++++ packer/buildfilter/filter_test.go | 174 +++++++++++ packer/core.go | 94 ++++++ packer/run_interfaces.go | 4 + website/content/docs/commands/build.mdx | 33 ++ .../hcl_templates/blocks/build/index.mdx | 25 ++ .../templates/hcl_templates/blocks/source.mdx | 23 ++ 17 files changed, 1013 insertions(+), 10 deletions(-) create mode 100644 command/test-fixtures/hcl-filter/build.pkr.hcl create mode 100644 packer/buildfilter/filter.go create mode 100644 packer/buildfilter/filter_test.go diff --git a/command/build.go b/command/build.go index 3f49634fb..9b9825ac9 100644 --- a/command/build.go +++ b/command/build.go @@ -138,6 +138,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int builds, diags := packerStarter.GetBuilds(packer.GetBuildsOptions{ Only: cla.Only, Except: cla.Except, + Filters: cla.Filters, Debug: cla.Debug, Force: cla.Force, OnError: cla.OnError, @@ -466,6 +467,13 @@ Options: -debug Debug mode enabled for builds. -except=foo,bar,baz Run all builds and post-processors other than these. -only=foo,bar,baz Build only the specified builds. + -filter='tags=prod,x86' Select builds by declared tags/labels. Repeatable; + multiple -filter flags are AND-ed together. + Grammar: KEY=VAL(,VAL) all of + KEY~=VAL(,VAL) any of + KEY!=VAL(,VAL) none of + KEY is either "tags" or a label key. Values are + globs. Applied after -only and -except. -force Force a build to continue if artifacts exist, deletes existing artifacts. -machine-readable Produce machine-readable output. -on-error=[cleanup|abort|ask|run-cleanup-provisioner] If the build fails do: clean up (default), abort, ask, or run-cleanup-provisioner. @@ -496,6 +504,7 @@ func (*BuildCommand) AutocompleteFlags() complete.Flags { "-debug": complete.PredictNothing, "-except": complete.PredictNothing, "-only": complete.PredictNothing, + "-filter": complete.PredictNothing, "-force": complete.PredictNothing, "-machine-readable": complete.PredictNothing, "-on-error": complete.PredictNothing, diff --git a/command/build_test.go b/command/build_test.go index 70425c666..b34d1d5e4 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -281,6 +281,64 @@ func TestBuild(t *testing.T) { }, }, + // -filter: tags and labels (HCL2) + { + name: "hcl - '-filter tags=prod' selects sources tagged prod", + args: []string{ + "-filter=tags=prod", + testFixture("hcl-filter"), + }, + fileCheck: fileCheck{ + expected: []string{"chocolate.txt", "cherry.txt"}, + notExpected: []string{"vanilla.txt"}, + }, + }, + { + name: "hcl - '-filter tags=prod,x86' selects intersection", + args: []string{ + "-filter=tags=prod,x86", + testFixture("hcl-filter"), + }, + fileCheck: fileCheck{ + expected: []string{"chocolate.txt"}, + notExpected: []string{"vanilla.txt", "cherry.txt"}, + }, + }, + { + name: "hcl - '-filter tags!=dev' excludes dev-tagged sources", + args: []string{ + "-filter=tags!=dev", + testFixture("hcl-filter"), + }, + fileCheck: fileCheck{ + expected: []string{"chocolate.txt", "cherry.txt"}, + notExpected: []string{"vanilla.txt"}, + }, + }, + { + name: "hcl - '-filter region~=us-*' selects by label glob", + args: []string{ + "-filter=region~=us-*", + testFixture("hcl-filter"), + }, + fileCheck: fileCheck{ + expected: []string{"chocolate.txt", "vanilla.txt"}, + notExpected: []string{"cherry.txt"}, + }, + }, + { + name: "hcl - multiple -filter flags are AND-ed", + args: []string{ + "-filter=tags=prod", + "-filter=region=us-east", + testFixture("hcl-filter"), + }, + fileCheck: fileCheck{ + expected: []string{"chocolate.txt"}, + notExpected: []string{"vanilla.txt", "cherry.txt"}, + }, + }, + // recipes { name: "hcl - recipes", diff --git a/command/cli.go b/command/cli.go index 643862e54..4cfe0feb1 100644 --- a/command/cli.go +++ b/command/cli.go @@ -56,6 +56,7 @@ func (ma *MetaArgs) GetConfigType() (configType, error) { func (ma *MetaArgs) AddFlagSets(fs *flag.FlagSet) { fs.Var((*sliceflag.StringFlag)(&ma.Only), "only", "") fs.Var((*sliceflag.StringFlag)(&ma.Except), "except", "") + fs.Var((*sliceflag.StringFlag)(&ma.Filters), "filter", "") fs.Var((*kvflag.Flag)(&ma.Vars), "var", "") fs.Var((*kvflag.StringSlice)(&ma.VarFiles), "var-file", "") fs.Var(&ma.ConfigType, "config-type", "set to 'hcl2' to run in hcl2 mode when no file is passed.") @@ -68,8 +69,11 @@ type MetaArgs struct { Path string Paths []string Only, Except []string - Vars map[string]string - VarFiles []string + // Filters holds repeated -filter expressions that select builds by + // tags/labels. See packer/buildfilter for the grammar. + Filters []string + Vars map[string]string + VarFiles []string // set to "hcl2" to force hcl2 mode ConfigType configType @@ -117,6 +121,7 @@ func GetCleanedBuildArgs(ba *BuildArgs) map[string]interface{} { "force": ba.Force, "only": ba.Only, "except": ba.Except, + "filter": ba.Filters, "var-files": ba.VarFiles, "path": ba.Path, } diff --git a/command/test-fixtures/hcl-filter/build.pkr.hcl b/command/test-fixtures/hcl-filter/build.pkr.hcl new file mode 100644 index 000000000..17fd44fd8 --- /dev/null +++ b/command/test-fixtures/hcl-filter/build.pkr.hcl @@ -0,0 +1,38 @@ +source "file" "chocolate" { + content = "chocolate" + target = "chocolate.txt" + metadata { + tags = ["prod", "x86"] + labels = { region = "us-east" } + } +} + +source "file" "vanilla" { + content = "vanilla" + target = "vanilla.txt" + metadata { + tags = ["dev", "x86"] + labels = { region = "us-west" } + } +} + +source "file" "cherry" { + content = "cherry" + target = "cherry.txt" + metadata { + tags = ["prod", "arm64"] + labels = { region = "eu-west" } + } +} + +build { + name = "my_build" + metadata { + tags = ["nightly"] + } + sources = [ + "file.chocolate", + "file.vanilla", + "file.cherry", + ] +} diff --git a/command/validate.go b/command/validate.go index 978954a9f..ca36b663f 100644 --- a/command/validate.go +++ b/command/validate.go @@ -91,8 +91,9 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int } _, diags = packerStarter.GetBuilds(packer.GetBuildsOptions{ - Only: cla.Only, - Except: cla.Except, + Only: cla.Only, + Except: cla.Except, + Filters: cla.Filters, }) fixerDiags := packerStarter.FixConfig(packer.FixConfigOptions{ diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 4d37aaa94..d94b24bec 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -182,7 +182,13 @@ func (cfg *PackerConfig) initializeBlocks() hcl.Diagnostics { continue } - body := sourceDefinition.block.Body + // Use sourceDefinition.Body (which has tags/labels stripped) + // rather than block.Body so plugin ConfigSpec decoding never + // sees the filter attributes. + body := sourceDefinition.Body + if body == nil { + body = sourceDefinition.block.Body + } if srcUsage.Body != nil { // merge additions into source definition to get a new body. body = hcl.MergeBodies([]hcl.Body{body, srcUsage.Body}) diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index a8b22ec34..4e5d180a5 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -37,6 +37,7 @@ var buildSchema = &hcl.BodySchema{ {Type: buildPostProcessorLabel, LabelNames: []string{"type"}}, {Type: buildPostProcessorsLabel, LabelNames: []string{}}, {Type: buildHCPPackerRegistryLabel}, + {Type: metadataBlockLabel}, }, } @@ -63,6 +64,17 @@ type BuildBlock struct { // call for example. Description string + // Tags is the list of tags declared on the build block. These are + // unioned with the per-source tags to produce the effective tag set + // used by the -filter CLI flag. + Tags []string + + // Labels is the key/value metadata declared on the build block. These + // are merged with the per-source labels to produce the effective + // label set used by the -filter CLI flag. On conflict, source labels + // take precedence over build labels because sources are more specific. + Labels map[string]string + // HCPPackerRegistry contains the configuration for publishing the image to the HCP Packer Registry. HCPPackerRegistry *HCPPackerRegistryBlock @@ -154,8 +166,27 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB if diags.HasErrors() { return nil, diags } + seenMetadata := false for _, block := range content.Blocks { switch block.Type { + case metadataBlockLabel: + if seenMetadata { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Only one %q block is allowed per build", metadataBlockLabel), + Subject: block.DefRange.Ptr(), + }) + continue + } + seenMetadata = true + var meta metadataBody + moreDiags := gohcl.DecodeBody(block.Body, cfg.EvalContext(LocalContext, nil), &meta) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + build.Tags = dedupStrings(meta.Tags) + build.Labels = meta.Labels case buildHCPPackerRegistryLabel: if build.HCPPackerRegistry != nil { diags = append(diags, &hcl.Diagnostic{ diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index d170649b9..526635199 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" pkrfunction "github.com/hashicorp/packer/hcl2template/function" "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/packer/buildfilter" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" ) @@ -711,6 +712,20 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]*packer.Core }) } + // Compile -filter expressions once up front so a parse error aborts + // before we start builder plugins. + filterExprs, err := buildfilter.Parse(opts.Filters) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid -filter expression", + Detail: err.Error(), + }) + return nil, diags + } + filterMatches := 0 + filterCandidates := 0 + for _, build := range cfg.Builds { for _, srcUsage := range build.Sources { src, found := cfg.Sources[srcUsage.SourceRef] @@ -727,6 +742,8 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]*packer.Core pcb := &packer.CoreBuild{ BuildName: build.Name, Type: srcUsage.String(), + Tags: mergeTags(build.Tags, src.Tags, srcUsage.Tags), + Labels: mergeLabels(build.Labels, srcUsage.Labels, src.Labels), } pcb.SetDebug(cfg.debug) @@ -776,6 +793,16 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]*packer.Core } } + // -filter: applied after -only/-except, against the effective + // tags/labels set already assigned to pcb. + if len(filterExprs) > 0 { + filterCandidates++ + if !buildfilter.Match(pcb, filterExprs) { + continue + } + filterMatches++ + } + builder, moreDiags, generatedVars := cfg.startBuilder(srcUsage, cfg.EvalContext(BuildContext, nil)) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { @@ -862,9 +889,65 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]*packer.Core "These could also be matched with a glob pattern like: 'happycloud.*'", possibleBuildNames), }) } + if len(filterExprs) > 0 && filterCandidates > 0 && filterMatches == 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "a -filter option was passed, but did not match any build.", + Detail: fmt.Sprintf("Possible build names: %v.\n"+ + "Verify that your source and build blocks declare the `tags` and/or `labels` attributes you are filtering on.", + possibleBuildNames), + }) + } return res, diags } +// mergeTags returns a deduplicated union of the tag slices, preserving the +// order in which values are first encountered (build tags first, then the +// source definition, then the inline source usage). The result is nil when +// every input is empty so CoreBuild.Tags reflects "no tags declared" as a +// nil rather than an empty non-nil slice. +func mergeTags(tagSets ...[]string) []string { + total := 0 + for _, s := range tagSets { + total += len(s) + } + if total == 0 { + return nil + } + seen := make(map[string]struct{}, total) + out := make([]string, 0, total) + for _, s := range tagSets { + for _, t := range s { + if _, ok := seen[t]; ok { + continue + } + seen[t] = struct{}{} + out = append(out, t) + } + } + return out +} + +// mergeLabels merges label maps. Earlier-listed maps provide defaults; +// later-listed maps override on key conflict. The caller passes them in +// precedence order (lowest to highest): typically build, usage, source. +func mergeLabels(labelSets ...map[string]string) map[string]string { + total := 0 + for _, m := range labelSets { + total += len(m) + } + if total == 0 { + return nil + } + out := make(map[string]string, total) + for _, m := range labelSets { + for k, v := range m { + out[k] = v + } + } + return out +} + var PackerConsoleHelp = strings.TrimSpace(` Packer console HCL2 Mode. The Packer console allows you to experiment with Packer interpolations. diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index e07a8f257..14d018157 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -25,6 +25,17 @@ type SourceBlock struct { block *hcl.Block + // Body is the source block body with the top-level `tags` and `labels` + // attributes stripped out. This is what downstream code (body merging, + // plugin decoding) should use. It is populated at parse time; if the + // source has no tags/labels it is equal to block.Body. + Body hcl.Body + + // Tags is the list of tags declared on the source block. + Tags []string + // Labels is the key/value metadata declared on the source block. + Labels map[string]string + // LocalName can be set in a singular source block from a build block, it // allows to give a special name to a build in the logs. LocalName string @@ -41,6 +52,16 @@ type SourceUseBlock struct { // allows to give a special name to a build in the logs. LocalName string + // Tags is the list of tags declared on an inline `source "type.name" {}` + // usage inside a build block. These are layered on top of the definition + // source's tags. + Tags []string + // Labels is the key/value metadata declared on an inline source usage + // inside a build block. Layered on top of the definition source's labels + // (usage keys win on conflict with the definition source, but the + // definition source still wins over the enclosing build block). + Labels map[string]string + // Rest of the body, in case the build.source block has more specific // content // Body can be expanded by a dynamic tag. @@ -66,27 +87,91 @@ func (b *SourceUseBlock) ctyValues() map[string]cty.Value { } } +// metadataBlockLabel is the reserved nested block name that carries +// Packer-specific build metadata (tags and labels) on source and build +// blocks. It is stripped from the body before the plugin ConfigSpec +// decoder runs, so it never collides with plugin-defined attributes such +// as amazon-ebs's own `tags` map. +const metadataBlockLabel = "metadata" + +// metadataBodySchema describes the attributes accepted inside a +// `metadata { }` nested block. +type metadataBody struct { + Tags []string `hcl:"tags,optional"` + Labels map[string]string `hcl:"labels,optional"` +} + +// extractMetadata pulls any `metadata` block out of body. It returns the +// extracted tags/labels, a remainder body with the metadata block removed, +// and any diagnostics. When body has no metadata block, tags and labels +// are nil and remainder == body. +func extractMetadata(body hcl.Body, ectx *hcl.EvalContext) (tags []string, labels map[string]string, remainder hcl.Body, diags hcl.Diagnostics) { + schema := &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + {Type: metadataBlockLabel}, + }, + } + content, remain, diags := body.PartialContent(schema) + if len(content.Blocks) == 0 { + // No metadata block: return the original body so callers can detect + // the no-op case by pointer equality and avoid perturbing downstream + // test fixtures. + return nil, nil, body, diags + } + remainder = remain + if len(content.Blocks) > 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Only one %q block is allowed per source or build", metadataBlockLabel), + Subject: content.Blocks[1].DefRange.Ptr(), + }) + } + mb := content.Blocks[0] + var decoded metadataBody + moreDiags := gohcl.DecodeBody(mb.Body, ectx, &decoded) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + return nil, nil, remainder, diags + } + return dedupStrings(decoded.Tags), decoded.Labels, remainder, diags +} + // decodeBuildSource reads a used source block from a build: // // build { // source "type.example" { // name = "local_name" +// metadata { +// tags = ["prod"] +// labels = { region = "us-east" } +// } // } // } func (p *Parser) decodeBuildSource(block *hcl.Block) (SourceUseBlock, hcl.Diagnostics) { ref := sourceRefFromString(block.Labels[0]) out := SourceUseBlock{SourceRef: ref} + + // First strip out the metadata block so the subsequent gohcl decode + // and the plugin ConfigSpec decoder never see it. + tags, labels, bodyAfterMeta, diags := extractMetadata(block.Body, nil) + if diags.HasErrors() { + return out, diags + } + out.Tags = tags + out.Labels = labels + var b struct { Name string `hcl:"name,optional"` Rest hcl.Body `hcl:",remain"` } - diags := gohcl.DecodeBody(block.Body, nil, &b) - if diags.HasErrors() { + moreDiags := gohcl.DecodeBody(bodyAfterMeta, nil, &b) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { return out, diags } out.LocalName = b.Name out.Body = b.Rest - return out, nil + return out, diags } func (p *Parser) decodeSource(block *hcl.Block) (SourceBlock, hcl.Diagnostics) { @@ -95,11 +180,39 @@ func (p *Parser) decodeSource(block *hcl.Block) (SourceBlock, hcl.Diagnostics) { Name: block.Labels[1], block: block, } - var diags hcl.Diagnostics - + tags, labels, remain, diags := extractMetadata(block.Body, nil) + // Only populate the filter-specific fields when the user actually + // declared a metadata block. Leaving Body/Tags/Labels as their zero + // values in the common case avoids perturbing equality checks in + // existing parser tests, and plugin.go falls back to block.Body when + // Body is nil. + if tags != nil || labels != nil { + source.Body = remain + source.Tags = tags + source.Labels = labels + } return source, diags } +// dedupStrings returns s with duplicates removed, preserving order. Returns +// nil when s is empty so callers can distinguish "no tags declared" from +// "empty tag list declared". +func dedupStrings(s []string) []string { + if len(s) == 0 { + return nil + } + seen := make(map[string]struct{}, len(s)) + out := make([]string, 0, len(s)) + for _, v := range s { + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + return out +} + func (cfg *PackerConfig) startBuilder(source SourceUseBlock, ectx *hcl.EvalContext) (packersdk.Builder, hcl.Diagnostics, []string) { var diags hcl.Diagnostics diff --git a/packer/build.go b/packer/build.go index de476ec67..b598ceff9 100644 --- a/packer/build.go +++ b/packer/build.go @@ -44,6 +44,16 @@ type CoreBuild struct { Variables map[string]string SensitiveVars []string + // Tags is the effective set of tags for this build, merged from the + // source block and the enclosing build block. Populated by the HCL2 and + // JSON template loaders. Consumed by the -filter CLI flag. + Tags []string + // Labels is the effective key/value metadata for this build, merged + // from the source block and the enclosing build block (source keys win + // on conflict). Populated by the HCL2 and JSON template loaders. + // Consumed by the -filter CLI flag. + Labels map[string]string + // Indicates whether the build is already initialized before calling Prepare(..) Prepared bool @@ -146,6 +156,14 @@ func (b *CoreBuild) Name() string { return b.Type } +// FilterTags returns the build's tag set for use by the buildfilter package. +// Satisfies buildfilter.Taggable. +func (b *CoreBuild) FilterTags() []string { return b.Tags } + +// FilterLabels returns the build's label map for use by the buildfilter +// package. Satisfies buildfilter.Taggable. +func (b *CoreBuild) FilterLabels() map[string]string { return b.Labels } + func (b *CoreBuild) packerConfig() map[string]interface{} { return map[string]interface{}{ common.BuildNameConfigKey: b.Type, diff --git a/packer/buildfilter/filter.go b/packer/buildfilter/filter.go new file mode 100644 index 000000000..11969242b --- /dev/null +++ b/packer/buildfilter/filter.go @@ -0,0 +1,288 @@ +// Copyright IBM Corp. 2013, 2025 +// SPDX-License-Identifier: BUSL-1.1 + +// Package buildfilter implements the parser and matcher for the +// `packer build -filter=...` flag. Filters select CoreBuilds by their +// declared tags and labels (key/value metadata). +// +// Grammar: +// +// := +// := "tags" | +// := "=" all of (every listed value must match) +// | "~=" any of (at least one listed value matches) +// | "!=" none of (no listed value matches) +// := ("," )* +// +// Values support glob patterns via github.com/gobwas/glob, matching the +// behavior of the existing -only/-except flags. +// +// Multiple -filter flags are AND-ed together, like Kubernetes label +// selectors: a build must satisfy every filter expression to be selected. +package buildfilter + +import ( + "fmt" + "strings" + + "github.com/gobwas/glob" +) + +// Op is a filter operator. +type Op int + +const ( + // OpAll requires every listed value to match (implicit AND within values). + OpAll Op = iota + // OpAny requires at least one listed value to match. + OpAny + // OpNone requires no listed value to match. + OpNone +) + +func (o Op) String() string { + switch o { + case OpAll: + return "=" + case OpAny: + return "~=" + case OpNone: + return "!=" + } + return "?" +} + +// Expr is a parsed filter expression. +type Expr struct { + // Key is either the literal string "tags" or a label key. + Key string + Op Op + // Values holds the compiled glob patterns the user supplied. + Values []glob.Glob + // Raw preserves the original user-supplied value list for error messages. + Raw []string +} + +// Taggable is the interface builds must satisfy to be filtered. CoreBuild +// implements this. +type Taggable interface { + FilterTags() []string + FilterLabels() map[string]string +} + +// Parse compiles a slice of raw filter strings (as supplied via -filter on +// the CLI) into a slice of Expr that can be evaluated with Match. +// +// An empty raw slice returns (nil, nil) and matches every build. +func Parse(raw []string) ([]Expr, error) { + if len(raw) == 0 { + return nil, nil + } + out := make([]Expr, 0, len(raw)) + for _, s := range raw { + e, err := parseOne(s) + if err != nil { + return nil, err + } + out = append(out, e) + } + return out, nil +} + +func parseOne(s string) (Expr, error) { + trimmed := strings.TrimSpace(s) + if trimmed == "" { + return Expr{}, fmt.Errorf("empty -filter expression") + } + + // Longer operator tokens must be tried first so "~=" and "!=" aren't + // mis-identified as "=". + var key, op, rest string + switch { + case containsOp(trimmed, "~="): + key, rest = splitOnce(trimmed, "~=") + op = "~=" + case containsOp(trimmed, "!="): + key, rest = splitOnce(trimmed, "!=") + op = "!=" + case containsOp(trimmed, "="): + key, rest = splitOnce(trimmed, "=") + op = "=" + default: + return Expr{}, fmt.Errorf("invalid -filter %q: expected KEY=VALUE, KEY~=VALUE, or KEY!=VALUE", s) + } + + key = strings.TrimSpace(key) + if key == "" { + return Expr{}, fmt.Errorf("invalid -filter %q: empty key", s) + } + if !validKey(key) { + return Expr{}, fmt.Errorf("invalid -filter key %q: must match [A-Za-z_][A-Za-z0-9_.-]*", key) + } + + rawValues := splitValues(rest) + if len(rawValues) == 0 { + return Expr{}, fmt.Errorf("invalid -filter %q: empty value list", s) + } + + globs := make([]glob.Glob, 0, len(rawValues)) + for _, v := range rawValues { + g, err := glob.Compile(v) + if err != nil { + return Expr{}, fmt.Errorf("invalid -filter %q: bad glob %q: %s", s, v, err) + } + globs = append(globs, g) + } + + e := Expr{Key: key, Values: globs, Raw: rawValues} + switch op { + case "=": + e.Op = OpAll + case "~=": + e.Op = OpAny + case "!=": + e.Op = OpNone + } + return e, nil +} + +// containsOp reports whether s contains op as a literal substring. A small +// helper to keep parseOne readable. +func containsOp(s, op string) bool { return strings.Contains(s, op) } + +// splitOnce splits s on the first occurrence of sep, returning the two halves. +func splitOnce(s, sep string) (string, string) { + i := strings.Index(s, sep) + if i < 0 { + return s, "" + } + return s[:i], s[i+len(sep):] +} + +// splitValues splits a comma-separated value list, trimming whitespace and +// dropping empty entries. +func splitValues(s string) []string { + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +func validKey(k string) bool { + if k == "" { + return false + } + for i, r := range k { + switch { + case r >= 'a' && r <= 'z': + case r >= 'A' && r <= 'Z': + case r == '_': + case i > 0 && (r >= '0' && r <= '9' || r == '.' || r == '-'): + default: + return false + } + } + return true +} + +// Match reports whether b satisfies every expression in exprs. An empty +// exprs slice matches any build. +func Match(b Taggable, exprs []Expr) bool { + if len(exprs) == 0 { + return true + } + tags := b.FilterTags() + labels := b.FilterLabels() + for _, e := range exprs { + if !matchOne(e, tags, labels) { + return false + } + } + return true +} + +func matchOne(e Expr, tags []string, labels map[string]string) bool { + if e.Key == "tags" { + return matchTags(e, tags) + } + return matchLabel(e, labels[e.Key]) +} + +func matchTags(e Expr, tags []string) bool { + anyHit := func(g glob.Glob) bool { + for _, t := range tags { + if g.Match(t) { + return true + } + } + return false + } + switch e.Op { + case OpAll: + for _, g := range e.Values { + if !anyHit(g) { + return false + } + } + return true + case OpAny: + for _, g := range e.Values { + if anyHit(g) { + return true + } + } + return false + case OpNone: + for _, g := range e.Values { + if anyHit(g) { + return false + } + } + return true + } + return false +} + +func matchLabel(e Expr, v string) bool { + // Label equality/inequality: the label value must match the supplied + // glob(s). A build with no such label fails OpAll/OpAny but satisfies + // OpNone (there is nothing to exclude it on). + switch e.Op { + case OpAll: + if v == "" { + return false + } + for _, g := range e.Values { + if !g.Match(v) { + return false + } + } + return true + case OpAny: + if v == "" { + return false + } + for _, g := range e.Values { + if g.Match(v) { + return true + } + } + return false + case OpNone: + if v == "" { + return true + } + for _, g := range e.Values { + if g.Match(v) { + return false + } + } + return true + } + return false +} diff --git a/packer/buildfilter/filter_test.go b/packer/buildfilter/filter_test.go new file mode 100644 index 000000000..57483f5a4 --- /dev/null +++ b/packer/buildfilter/filter_test.go @@ -0,0 +1,174 @@ +// Copyright IBM Corp. 2013, 2025 +// SPDX-License-Identifier: BUSL-1.1 + +package buildfilter + +import ( + "testing" +) + +type fakeBuild struct { + tags []string + labels map[string]string +} + +func (f fakeBuild) FilterTags() []string { return f.tags } +func (f fakeBuild) FilterLabels() map[string]string { return f.labels } + +func TestParse_Valid(t *testing.T) { + cases := []struct { + in string + op Op + key string + values []string + }{ + {"tags=prod,x86", OpAll, "tags", []string{"prod", "x86"}}, + {"tags~=prod,staging", OpAny, "tags", []string{"prod", "staging"}}, + {"tags!=experimental", OpNone, "tags", []string{"experimental"}}, + {"region=us-east", OpAll, "region", []string{"us-east"}}, + {"region~=us-*", OpAny, "region", []string{"us-*"}}, + {" region = us-east ", OpAll, "region", []string{"us-east"}}, + {"tier.sub_group=a,b", OpAll, "tier.sub_group", []string{"a", "b"}}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + exprs, err := Parse([]string{tc.in}) + if err != nil { + t.Fatalf("Parse(%q) unexpected err: %v", tc.in, err) + } + if len(exprs) != 1 { + t.Fatalf("Parse(%q) got %d exprs, want 1", tc.in, len(exprs)) + } + e := exprs[0] + if e.Key != tc.key { + t.Errorf("key: got %q, want %q", e.Key, tc.key) + } + if e.Op != tc.op { + t.Errorf("op: got %v, want %v", e.Op, tc.op) + } + if len(e.Raw) != len(tc.values) { + t.Fatalf("values len: got %d (%v), want %d (%v)", len(e.Raw), e.Raw, len(tc.values), tc.values) + } + for i := range tc.values { + if e.Raw[i] != tc.values[i] { + t.Errorf("value[%d]: got %q, want %q", i, e.Raw[i], tc.values[i]) + } + } + }) + } +} + +func TestParse_Invalid(t *testing.T) { + cases := []string{ + "", + "tags", + "=value", + "tags=", + "1bad=x", + "bad key=x", + } + for _, in := range cases { + t.Run(in, func(t *testing.T) { + if _, err := Parse([]string{in}); err == nil { + t.Fatalf("Parse(%q) expected error, got nil", in) + } + }) + } +} + +func TestParse_MultipleANDed(t *testing.T) { + exprs, err := Parse([]string{"tags=prod", "region=us-east"}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(exprs) != 2 { + t.Fatalf("got %d exprs, want 2", len(exprs)) + } +} + +func TestMatch_TagsAll(t *testing.T) { + exprs, _ := Parse([]string{"tags=prod,x86"}) + match := func(tags ...string) bool { + return Match(fakeBuild{tags: tags}, exprs) + } + if !match("prod", "x86", "us-east") { + t.Errorf("should match superset") + } + if match("prod") { + t.Errorf("missing x86 should fail") + } + if match("dev", "x86") { + t.Errorf("missing prod should fail") + } +} + +func TestMatch_TagsAny(t *testing.T) { + exprs, _ := Parse([]string{"tags~=prod,staging"}) + match := func(tags ...string) bool { + return Match(fakeBuild{tags: tags}, exprs) + } + if !match("prod", "x86") { + t.Errorf("prod should match") + } + if !match("staging") { + t.Errorf("staging should match") + } + if match("dev") { + t.Errorf("dev should not match") + } +} + +func TestMatch_TagsNone(t *testing.T) { + exprs, _ := Parse([]string{"tags!=experimental,broken"}) + match := func(tags ...string) bool { + return Match(fakeBuild{tags: tags}, exprs) + } + if !match("prod") { + t.Errorf("prod should match (no excluded tags)") + } + if match("prod", "broken") { + t.Errorf("broken should exclude") + } +} + +func TestMatch_LabelsGlob(t *testing.T) { + exprs, _ := Parse([]string{"region~=us-*"}) + if !Match(fakeBuild{labels: map[string]string{"region": "us-east"}}, exprs) { + t.Errorf("us-east should match us-*") + } + if Match(fakeBuild{labels: map[string]string{"region": "eu-west"}}, exprs) { + t.Errorf("eu-west should not match us-*") + } + if Match(fakeBuild{labels: nil}, exprs) { + t.Errorf("missing label should not satisfy ~= selector") + } +} + +func TestMatch_LabelsNoneOnMissing(t *testing.T) { + exprs, _ := Parse([]string{"region!=eu-*"}) + // build with no "region" label trivially satisfies != eu-*. + if !Match(fakeBuild{labels: nil}, exprs) { + t.Errorf("missing label should satisfy != selector") + } + if Match(fakeBuild{labels: map[string]string{"region": "eu-west"}}, exprs) { + t.Errorf("eu-west should be excluded by !=eu-*") + } +} + +func TestMatch_ANDSemantics(t *testing.T) { + exprs, _ := Parse([]string{"tags=prod", "region=us-east"}) + b1 := fakeBuild{tags: []string{"prod"}, labels: map[string]string{"region": "us-east"}} + if !Match(b1, exprs) { + t.Errorf("b1 should match both filters") + } + b2 := fakeBuild{tags: []string{"prod"}, labels: map[string]string{"region": "eu-west"}} + if Match(b2, exprs) { + t.Errorf("b2 should fail on region filter") + } +} + +func TestMatch_EmptyExprMatchesAll(t *testing.T) { + if !Match(fakeBuild{}, nil) { + t.Errorf("nil exprs should match any build") + } +} diff --git a/packer/core.go b/packer/core.go index 343e26c3b..fb66ea14b 100644 --- a/packer/core.go +++ b/packer/core.go @@ -25,6 +25,7 @@ import ( "github.com/hashicorp/packer-plugin-sdk/template" "github.com/hashicorp/packer-plugin-sdk/template/interpolate" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" + "github.com/hashicorp/packer/packer/buildfilter" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" packerversion "github.com/hashicorp/packer/version" "github.com/zclconf/go-cty/cty" @@ -401,6 +402,19 @@ func (c *Core) GetBuilds(opts GetBuildsOptions) ([]*CoreBuild, hcl.Diagnostics) buildNames := c.BuildNames(opts.Only, opts.Except) builds := []*CoreBuild{} diags := hcl.Diagnostics{} + + filterExprs, err := buildfilter.Parse(opts.Filters) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid -filter expression", + Detail: err.Error(), + }) + return nil, diags + } + filterCandidates := 0 + filterMatches := 0 + for _, n := range buildNames { b, err := c.Build(n) if err != nil { @@ -412,6 +426,16 @@ func (c *Core) GetBuilds(opts GetBuildsOptions) ([]*CoreBuild, hcl.Diagnostics) continue } + // -filter is applied before Prepare so skipped builds never run + // the plugin's Prepare code path. + if len(filterExprs) > 0 { + filterCandidates++ + if !buildfilter.Match(b, filterExprs) { + continue + } + filterMatches++ + } + // Now that build plugin has been launched, call Prepare() log.Printf("Preparing build: %s", b.Name()) b.SetDebug(opts.Debug) @@ -441,9 +465,76 @@ func (c *Core) GetBuilds(opts GetBuildsOptions) ([]*CoreBuild, hcl.Diagnostics) } } } + + if len(filterExprs) > 0 && filterCandidates > 0 && filterMatches == 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "a -filter option was passed, but did not match any build.", + Detail: "Verify that your JSON builders declare packer_tags and/or " + + "packer_labels matching the filter expression.", + }) + } + return builds, diags } +// extractJSONTagsLabels pulls Packer-reserved tags and labels out of a raw +// JSON builder config. The reserved keys are "packer_tags" (list of +// strings) and "packer_labels" (string-keyed string map). Using a +// packer_-prefix avoids collision with plugin-defined attributes such as +// amazon-ebs's own "tags" map for AWS AMI tagging. +// +// The keys are deleted from cfg in place so the plugin's Prepare call +// never sees them. Malformed values are silently ignored: JSON templates +// have no HCL diagnostics channel here and a malformed metadata field +// should not block a build that would otherwise succeed. +func extractJSONTagsLabels(cfg interface{}) ([]string, map[string]string) { + m, ok := cfg.(map[string]interface{}) + if !ok { + return nil, nil + } + + var tags []string + if raw, ok := m["packer_tags"]; ok { + if list, ok := raw.([]interface{}); ok { + for _, v := range list { + if s, ok := v.(string); ok { + tags = append(tags, s) + } + } + } + delete(m, "packer_tags") + } + + var labels map[string]string + if raw, ok := m["packer_labels"]; ok { + if lm, ok := raw.(map[string]interface{}); ok { + labels = make(map[string]string, len(lm)) + for k, v := range lm { + if s, ok := v.(string); ok { + labels[k] = s + } + } + } + delete(m, "packer_labels") + } + + // Dedup tags, preserving order. + if len(tags) > 0 { + seen := make(map[string]struct{}, len(tags)) + out := make([]string, 0, len(tags)) + for _, t := range tags { + if _, dup := seen[t]; dup { + continue + } + seen[t] = struct{}{} + out = append(out, t) + } + tags = out + } + return tags, labels +} + // Build returns the Build object for the given name. func (c *Core) Build(n string) (*CoreBuild, error) { // Setup the builder @@ -586,6 +677,7 @@ func (c *Core) Build(n string) (*CoreBuild, error) { // Return a structure that contains the plugins, their types, variables, and // the raw builder config loaded from the json template + tags, labels := extractJSONTagsLabels(configBuilder.Config) cb := &CoreBuild{ Type: n, Builder: builder, @@ -597,6 +689,8 @@ func (c *Core) Build(n string) (*CoreBuild, error) { TemplatePath: c.Template.Path, Variables: c.variables, SensitiveVars: sensitiveVars, + Tags: tags, + Labels: labels, } //configBuilder.Name is left uninterpolated so we must check against diff --git a/packer/run_interfaces.go b/packer/run_interfaces.go index c2dadc9b4..3b2ca6ace 100644 --- a/packer/run_interfaces.go +++ b/packer/run_interfaces.go @@ -13,6 +13,10 @@ type GetBuildsOptions struct { // Get builds except the ones that match with except and with only the ones // that match with Only. When those are empty everything matches. Except, Only []string + // Filters are repeated -filter expressions selecting builds by declared + // tags/labels. Applied after -only/-except. See packer/buildfilter for + // the grammar. An empty slice selects every build. + Filters []string Debug, Force bool OnError string diff --git a/website/content/docs/commands/build.mdx b/website/content/docs/commands/build.mdx index 09f0771cb..3ec90be68 100644 --- a/website/content/docs/commands/build.mdx +++ b/website/content/docs/commands/build.mdx @@ -56,6 +56,39 @@ artifacts that are created will be outputted at the end of the build. `@include 'commands/only.mdx'` +- `-filter='KEY=VALUE[,VALUE...]'` - Select builds by declared tags and labels. + Applied after `-only` and `-except`. Repeatable; multiple `-filter` flags are + AND-ed together (all must match for a build to run). + + The expression grammar is: + + - `KEY=VAL(,VAL)` - build must match **every** listed value (all-of) + - `KEY~=VAL(,VAL)` - build must match **at least one** listed value (any-of) + - `KEY!=VAL(,VAL)` - build must match **none** of the listed values (none-of) + + `KEY` is either the literal string `tags` (matching against the list of tags + declared on the source and build blocks) or a label key. Values are + [glob](https://github.com/gobwas/glob) patterns; for example + `-filter=region~=us-*` matches any `region` label starting with `us-`. + + Tags and labels are declared in HCL2 via a `metadata { ... }` block inside + a `source` or `build` block: + + ```hcl + source "amazon-ebs" "web" { + metadata { + tags = ["prod", "x86"] + labels = { region = "us-east" } + } + # ...builder-specific config + } + ``` + + In legacy JSON templates, use the reserved top-level builder keys + `packer_tags` (list of strings) and `packer_labels` (map of string to string). + Using the `packer_` prefix avoids collision with plugin-specific attributes + such as the `amazon-ebs` builder's own `tags` map. + - `-parallel-builds=N` - Limit the number of builds to run in parallel, 0 means no limit (defaults to 0). diff --git a/website/content/docs/templates/hcl_templates/blocks/build/index.mdx b/website/content/docs/templates/hcl_templates/blocks/build/index.mdx index 446ab8978..b069783a7 100644 --- a/website/content/docs/templates/hcl_templates/blocks/build/index.mdx +++ b/website/content/docs/templates/hcl_templates/blocks/build/index.mdx @@ -97,6 +97,31 @@ Here `'a.null.first-example'` was skipped. -> Note: It is not yet possible to match a named `build` block to do this, but this is soon going to be possible. So here "a.\*" will match nothing. +## `metadata` block + +A `build` block may declare a nested `metadata` block that carries Packer +build metadata used by the [`packer build -filter`](/packer/docs/commands/build) +flag. Two attributes are accepted: + +- `tags` (list of strings) — labels attached to every source selected by this build. +- `labels` (map of string to string) — key/value metadata merged with per-source labels. + +Build-level tags are unioned with per-source tags. Labels from the source +definition take precedence over labels declared on the enclosing build +block, so the more-specific source metadata wins on key conflict. + +```hcl +build { + metadata { + tags = ["nightly"] + } + sources = [ + "source.amazon-ebs.web", + "source.amazon-ebs.db", + ] +} +``` + ## Related - Refer to the [community builders reference](/packer/docs/community-tools#community-builders) for information about builders maintained by the community. diff --git a/website/content/docs/templates/hcl_templates/blocks/source.mdx b/website/content/docs/templates/hcl_templates/blocks/source.mdx index fdacbeed9..2d06f6033 100644 --- a/website/content/docs/templates/hcl_templates/blocks/source.mdx +++ b/website/content/docs/templates/hcl_templates/blocks/source.mdx @@ -57,6 +57,29 @@ build { `@include 'from-1.5/contextual-source-variables.mdx'` +## `metadata` block + +A source block may declare a nested `metadata` block that carries Packer +build metadata used by the [`packer build -filter`](/packer/docs/commands/build) +flag. Two attributes are accepted: + +- `tags` (list of strings) — labels attached to the source. +- `labels` (map of string to string) — key/value metadata attached to the source. + +The `metadata` block is reserved by Packer and is not passed to the builder +plugin, so it can never collide with builder-specific attributes such as +amazon-ebs's own `tags` map. + +```hcl +source "amazon-ebs" "web" { + metadata { + tags = ["prod", "x86"] + labels = { region = "us-east" } + } + # ...builder-specific config +} +``` + ## Related - Refer to the [builders reference overview](/packer/docs/builders) for information about Packer builders.