feat: Phase 2 Extension of Build Metadata (#13092)

command/build.go

@@ -113,6 +113,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 	if ret != 0 {
 		return ret
 	}
+	hcpRegistry.Metadata().Gather(GetCleanedBuildArgs(cla))
 
 	defer hcpRegistry.VersionStatusSummary()
 

command/cli.go

@@ -96,6 +96,30 @@ func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
 	ba.MetaArgs.AddFlagSets(flags)
 }
 
+// GetCleanedBuildArgs returns a map containing build flags specified to build for tracking within
+// the HCP Packer registry.
+//
+// Most of the arguments are kept as-is, except for the -var args, where only
+// the keys are kept to avoid leaking potential secrets.
+func GetCleanedBuildArgs(ba *BuildArgs) map[string]interface{} {
+	cleanedArgs := map[string]interface{}{
+		"debug":     ba.Debug,
+		"force":     ba.Force,
+		"only":      ba.Only,
+		"except":    ba.Except,
+		"var-files": ba.VarFiles,
+		"path":      ba.Path,
+	}
+
+	var varNames []string
+	for k := range ba.Vars {
+		varNames = append(varNames, k)
+	}
+	cleanedArgs["vars"] = varNames
+
+	return cleanedArgs
+}
+
 // BuildArgs represents a parsed cli line for a `packer build`
 type BuildArgs struct {
 	MetaArgs

internal/hcp/registry/hcl.go

@@ -21,6 +21,7 @@ type HCLRegistry struct {
 	configuration *hcl2template.PackerConfig
 	bucket        *Bucket
 	ui            sdkpacker.Ui
+	metadata      *MetadataStore
 }
 
 const (
@@ -87,8 +88,8 @@ func (h *HCLRegistry) CompleteBuild(
 		buildName = cb.Type
 	}
 
-	metadata := cb.GetMetadata()
-	err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, metadata)
+	buildMetadata, envMetadata := cb.GetMetadata(), h.metadata
+	err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata, envMetadata)
 	if err != nil {
 		return nil, err
 	}
@@ -164,5 +165,10 @@ func NewHCLRegistry(config *hcl2template.PackerConfig, ui sdkpacker.Ui) (*HCLReg
 		configuration: config,
 		bucket:        bucket,
 		ui:            ui,
+		metadata:      &MetadataStore{},
 	}, nil
 }
+
+func (h *HCLRegistry) Metadata() Metadata {
+	return h.metadata
+}

internal/hcp/registry/json.go

@@ -20,6 +20,7 @@ type JSONRegistry struct {
 	configuration *packer.Core
 	bucket        *Bucket
 	ui            sdkpacker.Ui
+	metadata      *MetadataStore
 }
 
 func NewJSONRegistry(config *packer.Core, ui sdkpacker.Ui) (*JSONRegistry, hcl.Diagnostics) {
@@ -52,6 +53,7 @@ func NewJSONRegistry(config *packer.Core, ui sdkpacker.Ui) (*JSONRegistry, hcl.D
 		configuration: config,
 		bucket:        bucket,
 		ui:            ui,
+		metadata:      &MetadataStore{},
 	}, nil
 }
 
@@ -95,8 +97,8 @@ func (h *JSONRegistry) CompleteBuild(
 	buildErr error,
 ) ([]sdkpacker.Artifact, error) {
 	buildName := build.Name()
-	buildMetadata := build.(*packer.CoreBuild).GetMetadata()
-	err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata)
+	buildMetadata, envMetadata := build.(*packer.CoreBuild).GetMetadata(), h.metadata
+	err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata, envMetadata)
 	if err != nil {
 		return nil, err
 	}
@@ -107,3 +109,8 @@ func (h *JSONRegistry) CompleteBuild(
 func (h *JSONRegistry) VersionStatusSummary() {
 	h.bucket.Version.statusSummary(h.ui)
 }
+
+// Metadata gets the global metadata object that registers global settings
+func (h *JSONRegistry) Metadata() Metadata {
+	return h.metadata
+}

internal/hcp/registry/metadata/cicd.go

@@ -0,0 +1,104 @@
+package metadata
+
+import (
+	"fmt"
+	"os"
+)
+
+type GithubActions struct{}
+
+func (g *GithubActions) Detect() error {
+	_, ok := os.LookupEnv("GITHUB_ACTIONS")
+	if !ok {
+		return fmt.Errorf("GITHUB_ACTIONS environment variable not found")
+	}
+	return nil
+}
+
+func (g *GithubActions) Details() map[string]interface{} {
+	env := make(map[string]interface{})
+	keys := []string{
+		"GITHUB_REPOSITORY",
+		"GITHUB_REPOSITORY_ID",
+		"GITHUB_WORKFLOW_URL",
+		"GITHUB_SHA",
+		"GITHUB_REF",
+		"GITHUB_ACTOR",
+		"GITHUB_ACTOR_ID",
+		"GITHUB_TRIGGERING_ACTOR",
+		"GITHUB_EVENT_NAME",
+		"GITHUB_JOB",
+	}
+
+	for _, key := range keys {
+		if value, ok := os.LookupEnv(key); ok {
+			env[key] = value
+		}
+	}
+
+	env["GITHUB_WORKFLOW_URL"] = fmt.Sprintf("%s/%s/actions/runs/%s", os.Getenv("GITHUB_SERVER_URL"), os.Getenv("GITHUB_REPOSITORY"), os.Getenv("GITHUB_RUN_ID"))
+	return env
+}
+
+func (g *GithubActions) Type() string {
+	return "github-actions"
+}
+
+type GitlabCI struct{}
+
+func (g *GitlabCI) Detect() error {
+	_, ok := os.LookupEnv("GITLAB_CI")
+	if !ok {
+		return fmt.Errorf("GITLAB_CI environment variable not found")
+	}
+	return nil
+}
+
+func (g *GitlabCI) Details() map[string]interface{} {
+	env := make(map[string]interface{})
+	keys := []string{
+		"CI_PROJECT_NAME",
+		"CI_PROJECT_ID",
+		"CI_PROJECT_URL",
+		"CI_COMMIT_SHA",
+		"CI_COMMIT_REF_NAME",
+		"GITLAB_USER_NAME",
+		"GITLAB_USER_ID",
+		"CI_PIPELINE_SOURCE",
+		"CI_PIPELINE_URL",
+		"CI_JOB_URL",
+		"CI_SERVER_NAME",
+		"CI_REGISTRY_IMAGE",
+	}
+
+	for _, key := range keys {
+		if value, ok := os.LookupEnv(key); ok {
+			env[key] = value
+		}
+	}
+
+	return env
+}
+
+func (g *GitlabCI) Type() string {
+	return "gitlab-ci"
+}
+
+func GetCicdMetadata() map[string]interface{} {
+	cicd := []MetadataProvider{
+		&GithubActions{},
+		&GitlabCI{},
+	}
+
+	for _, c := range cicd {
+		err := c.Detect()
+		if err == nil {
+			return map[string]interface{}{
+				"type":    c.Type(),
+				"details": c.Details(),
+			}
+		}
+	}
+
+	return nil
+}

internal/hcp/registry/metadata/os.go

@@ -0,0 +1,132 @@
+package metadata
+
+import (
+	"log"
+	"os/exec"
+	"runtime"
+	"strings"
+	"time"
+)
+
+type OSInfo struct {
+	Name    string
+	Arch    string
+	Version string
+}
+
+// CommandExecutor is an interface for executing commands.
+type CommandExecutor interface {
+	Exec(name string, arg ...string) ([]byte, error)
+}
+
+// DefaultExecutor is the default implementation of CommandExecutor.
+type DefaultExecutor struct{}
+
+// Exec executes a command and returns the combined output.
+func (d DefaultExecutor) Exec(name string, arg ...string) ([]byte, error) {
+	cmd := exec.Command(name, arg...)
+	return cmd.CombinedOutput()
+}
+
+var executor CommandExecutor = DefaultExecutor{}
+
+func GetOSMetadata() map[string]interface{} {
+	var osInfo OSInfo
+
+	switch runtime.GOOS {
+	case "windows":
+		osInfo = GetInfoForWindows(executor)
+	case "darwin":
+		osInfo = GetInfo(executor, "-srm")
+	case "linux":
+		osInfo = GetInfo(executor, "-srio")
+	case "freebsd":
+		osInfo = GetInfo(executor, "-sri")
+	case "openbsd":
+		osInfo = GetInfo(executor, "-srm")
+	case "netbsd":
+		osInfo = GetInfo(executor, "-srm")
+	default:
+		osInfo = OSInfo{
+			Name: runtime.GOOS,
+			Arch: runtime.GOARCH,
+		}
+	}
+
+	return map[string]interface{}{
+		"type": osInfo.Name,
+		"details": map[string]interface{}{
+			"arch":    osInfo.Arch,
+			"version": osInfo.Version,
+		},
+	}
+}
+
+func GetInfo(exec CommandExecutor, flags string) OSInfo {
+	out, err := uname(exec, flags)
+	tries := 0
+	for strings.Contains(out, "broken pipe") && tries < 3 {
+		out, err = uname(exec, flags)
+		time.Sleep(500 * time.Millisecond)
+		tries++
+	}
+	if strings.Contains(out, "broken pipe") || err != nil {
+		out = ""
+	}
+
+	if err != nil {
+		log.Printf("[ERROR] failed to get the OS info: %s", err)
+	}
+	core := retrieveCore(out)
+	return OSInfo{
+		Name:    runtime.GOOS,
+		Arch:    runtime.GOARCH,
+		Version: core,
+	}
+}
+
+func uname(exec CommandExecutor, flags string) (string, error) {
+	output, err := exec.Exec("uname", flags)
+	return string(output), err
+}
+
+func retrieveCore(osStr string) string {
+	osStr = strings.Replace(osStr, "\n", "", -1)
+	osStr = strings.Replace(osStr, "\r\n", "", -1)
+	osInfo := strings.Split(osStr, " ")
+
+	var core string
+	if len(osInfo) > 1 {
+		core = osInfo[1]
+	}
+	return core
+}
+
+func GetInfoForWindows(exec CommandExecutor) OSInfo {
+	out, err := exec.Exec("cmd", "ver")
+	if err != nil {
+		log.Printf("[ERROR] failed to get the OS info: %s", err)
+		return OSInfo{
+			Name: runtime.GOOS,
+			Arch: runtime.GOARCH,
+		}
+	}
+
+	osStr := strings.Replace(string(out), "\n", "", -1)
+	osStr = strings.Replace(osStr, "\r\n", "", -1)
+	tmp1 := strings.Index(osStr, "[Version")
+	tmp2 := strings.Index(osStr, "]")
+	var ver string
+	if tmp1 == -1 || tmp2 == -1 {
+		ver = ""
+	} else {
+		ver = osStr[tmp1+9 : tmp2]
+	}
+
+	osInfo := OSInfo{
+		Name:    runtime.GOOS,
+		Arch:    runtime.GOARCH,
+		Version: ver,
+	}
+	return osInfo
+}

internal/hcp/registry/metadata/os_test.go

@@ -0,0 +1,64 @@
+package metadata
+
+import (
+	"fmt"
+	"runtime"
+	"testing"
+)
+
+// MockExecutor is a mock implementation of CommandExecutor.
+type MockExecutor struct {
+	stdout string
+	err    error
+}
+
+// Exec returns a mocked output.
+func (m MockExecutor) Exec(name string, arg ...string) ([]byte, error) {
+	return []byte(m.stdout), m.err
+}
+
+func TestGetInfoForWindows(t *testing.T) {
+	tests := []struct {
+		name     string
+		stdout   string
+		err      error
+		expected OSInfo
+	}{
+		{
+			name:   "Valid version info",
+			stdout: "Microsoft Windows [Version 10.0.19042.928]",
+			err:    nil,
+			expected: OSInfo{
+				Name:    runtime.GOOS,
+				Arch:    runtime.GOARCH,
+				Version: "10.0.19042.928",
+			},
+		},
+		{
+			name:   "Invalid version info",
+			stdout: "Invalid output",
+			err:    fmt.Errorf("Invalid output"),
+			expected: OSInfo{
+				Name:    runtime.GOOS,
+				Arch:    runtime.GOARCH,
+				Version: "",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			mockExecutor := MockExecutor{
+				stdout: tt.stdout,
+				err:    tt.err,
+			}
+
+			result := GetInfoForWindows(mockExecutor)
+
+			if result != tt.expected {
+				t.Errorf("expected %+v, got %+v", tt.expected, result)
+			}
+		})
+	}
+}

internal/hcp/registry/metadata/vcs.go

@@ -0,0 +1,93 @@
+package metadata
+
+import (
+	"log"
+	"os"
+
+	gt "github.com/go-git/go-git/v5"
+)
+
+type MetadataProvider interface {
+	Detect() error
+	Details() map[string]interface{}
+	Type() string
+}
+
+type Git struct {
+	repo *gt.Repository
+}
+
+func (g *Git) Detect() error {
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Printf("[ERROR] unable to retrieve current directory: %s", err)
+		return err
+	}
+
+	repo, err := gt.PlainOpenWithOptions(wd, &gt.PlainOpenOptions{DetectDotGit: true})
+	if err != nil {
+		return err
+	}
+
+	g.repo = repo
+	return nil
+}
+
+func (g *Git) hasUncommittedChanges() bool {
+	worktree, err := g.repo.Worktree()
+	if err != nil {
+		log.Printf("[ERROR] failed to get the git worktree: %s", err)
+		return false
+	}
+
+	status, err := worktree.Status()
+	if err != nil {
+		log.Printf("[ERROR] failed to get the git worktree status: %s", err)
+		return false
+	}
+	return !status.IsClean()
+}
+
+func (g *Git) Type() string {
+	return "git"
+}
+
+func (g *Git) Details() map[string]interface{} {
+	resp := map[string]interface{}{}
+
+	headRef, err := g.repo.Head()
+	if err != nil {
+		log.Printf("[ERROR] failed to get the git branch name: %s", err)
+	} else {
+		resp["ref"] = headRef.Name().Short()
+	}
+
+	commit, err := g.repo.CommitObject(headRef.Hash())
+	if err != nil {
+		log.Printf("[ERROR] failed to get the git commit hash: %s", err)
+	} else {
+		resp["commit"] = commit.Hash.String()
+		resp["author"] = commit.Author.Name + " <" + commit.Author.Email + ">"
+	}
+
+	resp["has_uncommitted_changes"] = g.hasUncommittedChanges()
+	return resp
+}
+
+func GetVcsMetadata() map[string]interface{} {
+	vcsSystems := []MetadataProvider{
+		&Git{},
+	}
+
+	for _, vcs := range vcsSystems {
+		err := vcs.Detect()
+		if err == nil {
+			return map[string]interface{}{
+				"type":    vcs.Type(),
+				"details": vcs.Details(),
+			}
+		}
+	}
+
+	return nil
+}

internal/hcp/registry/null_registry.go

@@ -30,3 +30,7 @@ func (r nullRegistry) CompleteBuild(
 }
 
 func (r nullRegistry) VersionStatusSummary() {}
+
+func (r nullRegistry) Metadata() Metadata {
+	return NilMetadata{}
+}

internal/hcp/registry/registry.go

@@ -19,6 +19,7 @@ type Registry interface {
 	StartBuild(context.Context, sdkpacker.Build) error
 	CompleteBuild(ctx context.Context, build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error)
 	VersionStatusSummary()
+	Metadata() Metadata
 }
 
 // New instantiates the appropriate registry for the Packer configuration template type.

internal/hcp/registry/types.metadata_store.go

@@ -0,0 +1,39 @@
+package registry
+
+import "github.com/hashicorp/packer/internal/hcp/registry/metadata"
+
+// Metadata is the global metadata store, it is attached to a registry implementation
+// and keeps track of the environmental information.
+// This then can be sent to HCP Packer, so we can present it to users.
+type Metadata interface {
+	// Gather is the point where we vacuum all the information
+	// relevant from the environment in order to expose it to HCP Packer.
+	Gather(args map[string]interface{})
+}
+
+// MetadataStore is the effective implementation of a global store for metadata
+// destined to be uploaded to HCP Packer.
+//
+// If HCP is enabled during a build, this is populated with a curated list of
+// arguments to the build command, and environment-related information.
+type MetadataStore struct {
+	PackerBuildCommandOptions map[string]interface{}
+	OperatingSystem           map[string]interface{}
+	Vcs                       map[string]interface{}
+	Cicd                      map[string]interface{}
+}
+
+func (ms *MetadataStore) Gather(args map[string]interface{}) {
+	ms.OperatingSystem = metadata.GetOSMetadata()
+	ms.Cicd = metadata.GetCicdMetadata()
+	ms.Vcs = metadata.GetVcsMetadata()
+	ms.PackerBuildCommandOptions = args
+}
+
+// NilMetadata is a dummy implementation of a Metadata that does nothing.
+//
+// It is the implementation used typically when HCP is disabled, so nothing is
+// collected or kept in memory in this case.
+type NilMetadata struct{}
+
+func (ns NilMetadata) Gather(args map[string]interface{}) {}

internal/hcp/registry/types.version.go

@@ -179,7 +179,7 @@ func (version *Version) statusSummary(ui sdkpacker.Ui) {
 
 // AddMetadataToBuild adds metadata to a build in the HCP Packer registry.
 func (version *Version) AddMetadataToBuild(
-	ctx context.Context, buildName string, metadata packer.BuildMetadata,
+	ctx context.Context, buildName string, buildMetadata packer.BuildMetadata, globalMetadata *MetadataStore,
 ) error {
 	buildToUpdate, err := version.Build(buildName)
 	if err != nil {
@@ -187,10 +187,10 @@ func (version *Version) AddMetadataToBuild(
 	}
 
 	packerMetadata := make(map[string]interface{})
-	packerMetadata["version"] = metadata.PackerVersion
+	packerMetadata["version"] = buildMetadata.PackerVersion
 
 	var pluginsMetadata []map[string]interface{}
-	for _, plugin := range metadata.Plugins {
+	for _, plugin := range buildMetadata.Plugins {
 		pluginMetadata := map[string]interface{}{
 			"version": plugin.Description.Version,
 			"name":    plugin.Name,
@@ -198,7 +198,12 @@ func (version *Version) AddMetadataToBuild(
 		pluginsMetadata = append(pluginsMetadata, pluginMetadata)
 	}
 	packerMetadata["plugins"] = pluginsMetadata
+	packerMetadata["options"] = globalMetadata.PackerBuildCommandOptions
+	packerMetadata["os"] = globalMetadata.OperatingSystem
 
 	buildToUpdate.Metadata.Packer = packerMetadata
+	buildToUpdate.Metadata.Vcs = globalMetadata.Vcs
+	buildToUpdate.Metadata.Cicd = globalMetadata.Cicd
+
 	return nil
 }