[VAULT-39996] pipeline(sync): add support for checking changed files (#12220) (#12296)

Signed-off-by: Ryan Cragun <me@ryan.ec>
Co-authored-by: Ryan Cragun <me@ryan.ec>

.hooks/pre-push

@@ -97,7 +97,7 @@ main() {
 
     local output
     if ! output=$(builtin type pipeline 2>&1); then
-      fail "Unable to locate the 'pipeline' binary your \$PATH" "Please make sure you have the latest pipeline tool installed and that your '\$GOBIN' is in your '\$PATH'. You can use 'make tools-pipeline' to build and install the latest version of the 'pipeline' tool. The tool is required to check your branch's changed files before pushing to either 'hashicorp/vault' or 'ce/*\' branches in 'hashicorp/vault-enterprise'. Reach out to #team-vault-automation') for help! ${output}"
+      fail "Unable to locate the 'pipeline' binary your \$PATH" "Please make sure you have the latest pipeline tool installed and that your '\$GOBIN' is in your '\$PATH'. You can use 'make tools-pipeline' to build and install the latest version of the 'pipeline' tool. The tool is required to check your branch's changed files before pushing to either 'hashicorp/vault' or 'ce/*\' branches in 'hashicorp/vault-enterprise'. Reach out to #team-vault-automation for help! ${output}"
     fi
 
     # Determine our "zero" object ID.

tools/pipeline/internal/cmd/git_check_changed_files.go

@@ -15,7 +15,7 @@ var checkGitChangedFiles = &git.CheckChangedFilesReq{}
 
 func newGitCheckChangedFilesCmd() *cobra.Command {
 	changedFilesCmd := &cobra.Command{
-		Use:   "changed-files [--branch <branch> | --range <range> | --commit <sha>] --group <group>...",
+		Use:   "changed-files [--branch <branch> | --range <range> | --commit <sha>] --disallowed-group <group>...",
 		Short: "Check if any changed files are matching disallowed groups",
 		Long:  "Check if any changed files are matching disallowed groups",
 		RunE:  runGitCheckChangedFilesCmd,
@@ -25,7 +25,7 @@ func newGitCheckChangedFilesCmd() *cobra.Command {
 	changedFilesCmd.PersistentFlags().StringVarP(&checkGitChangedFiles.Branch, "branch", "b", "", "The branch to compare against")
 	changedFilesCmd.PersistentFlags().StringVarP(&checkGitChangedFiles.Range, "range", "r", "", "The commit range to compare (e.g., HEAD~5..HEAD)")
 	changedFilesCmd.PersistentFlags().StringVarP(&checkGitChangedFiles.Commit, "commit", "c", "", "The specific commit SHA to analyze")
-	changedFilesCmd.PersistentFlags().StringSliceVarP(&checkGitChangedFiles.CheckGroups, "groups", "g", nil, "File group(s) to check changed files for")
+	changedFilesCmd.PersistentFlags().StringSliceVarP(&checkGitChangedFiles.CheckGroups, "disallowed-groups", "g", nil, "File group(s) to check changed files for")
 	changedFilesCmd.PersistentFlags().BoolVar(&checkGitChangedFiles.WriteToGithubOutput, "github-output", false, "Whether or not to write 'changed-files' to $GITHUB_OUTPUT")
 
 	return changedFilesCmd

tools/pipeline/internal/cmd/github_sync_branch.go

@@ -31,6 +31,7 @@ func newSyncGithubBranchCmd() *cobra.Command {
 	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToRepo, "to-repo", "vault", "The Github repository to sync to")
 	syncBranchCmd.PersistentFlags().StringVar(&syncGithubBranchReq.ToBranch, "to-branch", "", "The name of the branch we want to sync to")
 	syncBranchCmd.PersistentFlags().StringVarP(&syncGithubBranchReq.RepoDir, "repo-dir", "d", "", "The path to the vault repository dir. If not set a temporary directory will be used")
+	syncBranchCmd.PersistentFlags().StringSliceVarP(&syncGithubBranchReq.CheckGroups, "disallowed-groups", "g", nil, "Enable changed file group and disallow if any files match the given groups")
 
 	err := syncBranchCmd.MarkPersistentFlagRequired("from-branch")
 	if err != nil {

tools/pipeline/internal/pkg/github/sync_branch_request.go

@@ -11,9 +11,11 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"strings"
 
 	libgithub "github.com/google/go-github/v81/github"
-	libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git/client"
+	gitpkg "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git"
+	gitclient "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git/client"
 	"github.com/jedib0t/go-pretty/v6/table"
 	slogctx "github.com/veqryn/slog-context"
 )
@@ -23,6 +25,7 @@ import (
 //
 // NOTE: We require that both branches exist for the operation to succeed.
 type SyncBranchReq struct {
+	// Repository and branch information
 	FromOwner  string
 	FromRepo   string
 	FromOrigin string
@@ -32,6 +35,8 @@ type SyncBranchReq struct {
 	ToOrigin   string
 	ToBranch   string
 	RepoDir    string
+	// Optional changed files checking
+	CheckGroups []string
 }
 
 // SyncBranchRes is a copy pull request response.
@@ -44,11 +49,12 @@ type SyncBranchRes struct {
 func (r *SyncBranchReq) Run(
 	ctx context.Context,
 	github *libgithub.Client,
-	git *libgit.Client,
+	git *gitclient.Client,
 ) (*SyncBranchRes, error) {
 	var err error
 	res := &SyncBranchRes{Request: r}
 
+	checkGroupsStr := strings.Join(r.CheckGroups, ", ")
 	slog.Default().DebugContext(slogctx.Append(ctx,
 		slog.String("from-owner", r.FromOwner),
 		slog.String("from-repo", r.FromRepo),
@@ -59,6 +65,7 @@ func (r *SyncBranchReq) Run(
 		slog.String("to-origin", r.ToOrigin),
 		slog.String("to-branch", r.ToBranch),
 		slog.String("repo-dir", r.RepoDir),
+		slog.String("disallowed-groups", checkGroupsStr),
 	), "synchronizing branches")
 
 	// Make sure we have required and valid fields
@@ -97,7 +104,7 @@ func (r *SyncBranchReq) Run(
 	// Check out our branch. Our intialization above will ensure we have a local
 	// reference.
 	slog.Default().DebugContext(ctx, "checking out to-branch")
-	checkoutRes, err := git.Checkout(ctx, &libgit.CheckoutOpts{
+	checkoutRes, err := git.Checkout(ctx, &gitclient.CheckoutOpts{
 		Branch: r.ToBranch,
 	})
 	if err != nil {
@@ -106,8 +113,8 @@ func (r *SyncBranchReq) Run(
 
 	// Add our from upstream as a remote and fetch our from branch.
 	slog.Default().DebugContext(ctx, "adding from upstream and fetching from-branch")
-	remoteRes, err := git.Remote(ctx, &libgit.RemoteOpts{
-		Command: libgit.RemoteCommandAdd,
+	remoteRes, err := git.Remote(ctx, &gitclient.RemoteOpts{
+		Command: gitclient.RemoteCommandAdd,
 		Track:   []string{r.FromBranch},
 		Fetch:   true,
 		Name:    r.FromOrigin,
@@ -118,15 +125,44 @@ func (r *SyncBranchReq) Run(
 		return res, err
 	}
 
-	// Use our remote reference as we haven't created a local reference.
+	// Use our remote reference as our fromBranch as we haven't created a local
+	// reference.
 	fromBranch := "remotes/" + r.FromOrigin + "/" + r.FromBranch
+
+	// Verify that our local branch does not contain and files that are in
+	// disallowed changed files groups.
+	if len(r.CheckGroups) > 0 {
+		slog.Default().DebugContext(ctx, "checking branch history for changed files in disallowed groups")
+
+		checkChangedFiles := gitpkg.CheckChangedFilesReq{
+			// Using the branch option here will inspect the entirely history of
+			// added files to the branch to ensure that we don't accidentally have
+			// some disallowed files in the branch history.
+			Branch:              fromBranch,
+			CheckGroups:         r.CheckGroups,
+			WriteToGithubOutput: false,
+		}
+
+		checkChangedFilesRes, err := checkChangedFiles.Run(ctx, git)
+		if err != nil {
+			return res, fmt.Errorf("checking branch history for changed files: %s: %w", checkChangedFilesRes.String(), err)
+		}
+
+		if l := len(checkChangedFilesRes.MatchedFiles); l > 0 {
+			return res, fmt.Errorf(
+				"found %d files that matched disallowed-groups %s: %s",
+				l, checkGroupsStr, strings.Join(checkChangedFilesRes.MatchedFiles.Names(), ", "),
+			)
+		}
+	}
+
 	slog.Default().DebugContext(ctx, "merging from-branch into to-branch")
-	mergeRes, err := git.Merge(ctx, &libgit.MergeOpts{
+	mergeRes, err := git.Merge(ctx, &gitclient.MergeOpts{
 		NoVerify: true,
-		Strategy: libgit.MergeStrategyORT,
-		StrategyOptions: []libgit.MergeStrategyOption{
-			libgit.MergeStrategyOptionTheirs,
-			libgit.MergeStrategyOptionIgnoreSpaceChange,
+		Strategy: gitclient.MergeStrategyORT,
+		StrategyOptions: []gitclient.MergeStrategyOption{
+			gitclient.MergeStrategyOptionTheirs,
+			gitclient.MergeStrategyOptionIgnoreSpaceChange,
 		},
 		IntoName: r.ToBranch,
 		Commit:   fromBranch,
@@ -136,7 +172,7 @@ func (r *SyncBranchReq) Run(
 	}
 
 	slog.Default().DebugContext(ctx, "pushing to-branch")
-	pushRes, err := git.Push(ctx, &libgit.PushOpts{
+	pushRes, err := git.Push(ctx, &gitclient.PushOpts{
 		Repository: r.ToOrigin,
 		Refspec:    []string{r.ToBranch},
 	})