From 31a5aa18788fa9fd7d3174ad71aea2bfe712127d Mon Sep 17 00:00:00 2001 From: Masayuki Morita Date: Wed, 10 Mar 2021 01:12:00 +0900 Subject: [PATCH] command/init: Add a new flag `-lockfile=readonly` (#27630) Fixes #27506 Add a new flag `-lockfile=readonly` to `terraform init`. It would be useful to allow us to suppress dependency lockfile changes explicitly. The type of the `-lockfile` flag is string rather than bool, leaving room for future extensions to other behavior variants. The readonly mode suppresses lockfile changes, but should verify checksums against the information already recorded. It should conflict with the `-upgrade` flag. Note: In the original use-case described in #27506, I would like to suppress adding zh hashes, but a test code here suppresses adding h1 hashes because it's easy for testing. Co-authored-by: Alisdair McDiarmid --- command/init.go | 38 ++++- command/init_test.go | 150 ++++++++++++++++++ .../main.tf | 10 ++ internal/depsfile/locks.go | 17 ++ internal/depsfile/locks_test.go | 58 +++++++ website/docs/cli/commands/init.html.md | 8 + 6 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 command/testdata/init-provider-lock-file-readonly-add/main.tf diff --git a/command/init.go b/command/init.go index d9726ff5f2..3931e97130 100644 --- a/command/init.go +++ b/command/init.go @@ -32,7 +32,7 @@ type InitCommand struct { } func (c *InitCommand) Run(args []string) int { - var flagFromModule string + var flagFromModule, flagLockfile string var flagBackend, flagGet, flagUpgrade bool var flagPluginPath FlagStringSlice flagConfigExtra := newRawFlags("-backend-config") @@ -47,6 +47,7 @@ func (c *InitCommand) Run(args []string) int { cmdFlags.BoolVar(&c.reconfigure, "reconfigure", false, "reconfigure") cmdFlags.BoolVar(&flagUpgrade, "upgrade", false, "") cmdFlags.Var(&flagPluginPath, "plugin-dir", "plugin directory") + cmdFlags.StringVar(&flagLockfile, "lockfile", "", "Set a dependency lockfile mode") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 @@ -260,7 +261,7 @@ func (c *InitCommand) Run(args []string) int { } // Now that we have loaded all modules, check the module tree for missing providers. - providersOutput, providersAbort, providerDiags := c.getProviders(config, state, flagUpgrade, flagPluginPath) + providersOutput, providersAbort, providerDiags := c.getProviders(config, state, flagUpgrade, flagPluginPath, flagLockfile) diags = diags.Append(providerDiags) if providersAbort || providerDiags.HasErrors() { c.showDiagnostics(diags) @@ -391,7 +392,7 @@ the backend configuration is present and valid. // Load the complete module tree, and fetch any missing providers. // This method outputs its own Ui. -func (c *InitCommand) getProviders(config *configs.Config, state *states.State, upgrade bool, pluginDirs []string) (output, abort bool, diags tfdiags.Diagnostics) { +func (c *InitCommand) getProviders(config *configs.Config, state *states.State, upgrade bool, pluginDirs []string, flagLockfile string) (output, abort bool, diags tfdiags.Diagnostics) { // Dev overrides cause the result of "terraform init" to be irrelevant for // any overridden providers, so we'll warn about it to avoid later // confusion when Terraform ends up using a different provider than the @@ -725,6 +726,11 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, mode := providercache.InstallNewProvidersOnly if upgrade { + if flagLockfile == "readonly" { + c.Ui.Error("The -upgrade flag conflicts with -lockfile=readonly.") + return true, true, diags + } + mode = providercache.InstallUpgrades } newLocks, err := inst.EnsureProviderVersions(ctx, previousLocks, reqs, mode) @@ -752,6 +758,28 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, // it's the smallest change relative to what came before it, which was // a hidden JSON file specifically for tracking providers.) if !newLocks.Equal(previousLocks) { + // if readonly mode + if flagLockfile == "readonly" { + // check if required provider dependences change + if !newLocks.EqualProviderAddress(previousLocks) { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + `Provider dependency changes detected`, + `Changes to the required provider dependencies were detected, but the lock file is read-only. To use and record these requirements, run "terraform init" without the "-lockfile=readonly" flag.`, + )) + return true, true, diags + } + + // suppress updating the file to record any new information it learned, + // such as a hash using a new scheme. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + `Provider lock file not updated`, + `Changes to the provider selections were detected, but not saved in the .terraform.lock.hcl file. To record these selections, run "terraform init" without the "-lockfile=readonly" flag.`, + )) + return true, false, diags + } + if previousLocks.Empty() { // A change from empty to non-empty is special because it suggests // we're running "terraform init" for the first time against a @@ -960,6 +988,10 @@ Options: -upgrade=false If installing modules (-get) or plugins, ignore previously-downloaded objects and install the latest version allowed within configured constraints. + + -lockfile=MODE Set a dependency lockfile mode. + Currently only "readonly" is valid. + ` return strings.TrimSpace(helpText) } diff --git a/command/init_test.go b/command/init_test.go index c0f19b3c3c..f55cc23354 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -1620,6 +1620,156 @@ provider "registry.terraform.io/hashicorp/test" { } } +func TestInit_providerLockFileReadonly(t *testing.T) { + // The hash in here is for the fake package that newMockProviderSource produces + // (so it'll change if newMockProviderSource starts producing different contents) + inputLockFile := strings.TrimSpace(` +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/test" { + version = "1.2.3" + constraints = "1.2.3" + hashes = [ + "zh:e919b507a91e23a00da5c2c4d0b64bcc7900b68d43b3951ac0f6e5d80387fbdc", + ] +} +`) + + badLockFile := strings.TrimSpace(` +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/test" { + version = "1.2.3" + constraints = "1.2.3" + hashes = [ + "zh:0000000000000000000000000000000000000000000000000000000000000000", + ] +} +`) + + updatedLockFile := strings.TrimSpace(` +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/test" { + version = "1.2.3" + constraints = "1.2.3" + hashes = [ + "h1:wlbEC2mChQZ2hhgUhl6SeVLPP7fMqOFUZAQhQ9GIIno=", + "zh:e919b507a91e23a00da5c2c4d0b64bcc7900b68d43b3951ac0f6e5d80387fbdc", + ] +} +`) + + cases := []struct { + desc string + fixture string + providers map[string][]string + input string + args []string + ok bool + want string + }{ + { + desc: "default", + fixture: "init-provider-lock-file", + providers: map[string][]string{"test": {"1.2.3"}}, + input: inputLockFile, + args: []string{}, + ok: true, + want: updatedLockFile, + }, + { + desc: "readonly", + fixture: "init-provider-lock-file", + providers: map[string][]string{"test": {"1.2.3"}}, + input: inputLockFile, + args: []string{"-lockfile=readonly"}, + ok: true, + want: inputLockFile, + }, + { + desc: "conflict", + fixture: "init-provider-lock-file", + providers: map[string][]string{"test": {"1.2.3"}}, + input: inputLockFile, + args: []string{"-lockfile=readonly", "-upgrade"}, + ok: false, + want: inputLockFile, + }, + { + desc: "checksum mismatch", + fixture: "init-provider-lock-file", + providers: map[string][]string{"test": {"1.2.3"}}, + input: badLockFile, + args: []string{"-lockfile=readonly"}, + ok: false, + want: badLockFile, + }, + { + desc: "reject to change required provider dependences", + fixture: "init-provider-lock-file-readonly-add", + providers: map[string][]string{ + "test": {"1.2.3"}, + "foo": {"1.0.0"}, + }, + input: inputLockFile, + args: []string{"-lockfile=readonly"}, + ok: false, + want: inputLockFile, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + // Create a temporary working directory that is empty + td := tempDir(t) + testCopyDir(t, testFixturePath(tc.fixture), td) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + providerSource, close := newMockProviderSource(t, tc.providers) + defer close() + + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + ProviderSource: providerSource, + } + + c := &InitCommand{ + Meta: m, + } + + // write input lockfile + lockFile := ".terraform.lock.hcl" + if err := ioutil.WriteFile(lockFile, []byte(tc.input), 0644); err != nil { + t.Fatalf("failed to write input lockfile: %s", err) + } + + code := c.Run(tc.args) + if tc.ok && code != 0 { + t.Fatalf("bad: \n%s", ui.ErrorWriter.String()) + } + if !tc.ok && code == 0 { + t.Fatalf("expected error, got output: \n%s", ui.OutputWriter.String()) + } + + buf, err := ioutil.ReadFile(lockFile) + if err != nil { + t.Fatalf("failed to read dependency lock file %s: %s", lockFile, err) + } + buf = bytes.TrimSpace(buf) + if diff := cmp.Diff(tc.want, string(buf)); diff != "" { + t.Errorf("wrong dependency lock file contents\n%s", diff) + } + }) + } +} + func TestInit_pluginDirReset(t *testing.T) { td := testTempDir(t) defer os.RemoveAll(td) diff --git a/command/testdata/init-provider-lock-file-readonly-add/main.tf b/command/testdata/init-provider-lock-file-readonly-add/main.tf new file mode 100644 index 0000000000..a706a53838 --- /dev/null +++ b/command/testdata/init-provider-lock-file-readonly-add/main.tf @@ -0,0 +1,10 @@ +terraform { + required_providers { + test = { + version = "1.2.3" + } + foo = { + version = "1.0.0" + } + } +} diff --git a/internal/depsfile/locks.go b/internal/depsfile/locks.go index a53994a6e5..e636522a17 100644 --- a/internal/depsfile/locks.go +++ b/internal/depsfile/locks.go @@ -203,6 +203,23 @@ func (l *Locks) Equal(other *Locks) bool { return true } +// EqualProviderAddress returns true if the given Locks have the same provider +// address as the receiver. This doesn't check version and hashes. +func (l *Locks) EqualProviderAddress(other *Locks) bool { + if len(l.providers) != len(other.providers) { + return false + } + + for addr := range l.providers { + _, ok := other.providers[addr] + if !ok { + return false + } + } + + return true +} + // Empty returns true if the given Locks object contains no actual locks. // // UI code might wish to use this to distinguish a lock file being diff --git a/internal/depsfile/locks_test.go b/internal/depsfile/locks_test.go index 1723113f97..9e319415d6 100644 --- a/internal/depsfile/locks_test.go +++ b/internal/depsfile/locks_test.go @@ -80,3 +80,61 @@ func TestLocksEqual(t *testing.T) { nonEqualBothWays(t, a, b) }) } + +func TestLocksEqualProviderAddress(t *testing.T) { + boopProvider := addrs.NewDefaultProvider("boop") + v2 := getproviders.MustParseVersion("2.0.0") + v2LocalBuild := getproviders.MustParseVersion("2.0.0+awesomecorp.1") + v2GtConstraints := getproviders.MustParseVersionConstraints(">= 2.0.0") + v2EqConstraints := getproviders.MustParseVersionConstraints("2.0.0") + hash1 := getproviders.HashScheme("test").New("1") + hash2 := getproviders.HashScheme("test").New("2") + hash3 := getproviders.HashScheme("test").New("3") + + equalProviderAddressBothWays := func(t *testing.T, a, b *Locks) { + t.Helper() + if !a.EqualProviderAddress(b) { + t.Errorf("a should be equal to b") + } + if !b.EqualProviderAddress(a) { + t.Errorf("b should be equal to a") + } + } + nonEqualProviderAddressBothWays := func(t *testing.T, a, b *Locks) { + t.Helper() + if a.EqualProviderAddress(b) { + t.Errorf("a should be equal to b") + } + if b.EqualProviderAddress(a) { + t.Errorf("b should be equal to a") + } + } + + t.Run("both empty", func(t *testing.T) { + a := NewLocks() + b := NewLocks() + equalProviderAddressBothWays(t, a, b) + }) + t.Run("an extra provider lock", func(t *testing.T) { + a := NewLocks() + b := NewLocks() + b.SetProvider(boopProvider, v2, v2GtConstraints, nil) + nonEqualProviderAddressBothWays(t, a, b) + }) + t.Run("both have boop provider with different versions", func(t *testing.T) { + a := NewLocks() + b := NewLocks() + a.SetProvider(boopProvider, v2, v2EqConstraints, nil) + b.SetProvider(boopProvider, v2LocalBuild, v2EqConstraints, nil) + equalProviderAddressBothWays(t, a, b) + }) + t.Run("both have boop provider with same version but different hashes", func(t *testing.T) { + a := NewLocks() + b := NewLocks() + hashesA := []getproviders.Hash{hash1, hash2} + hashesB := []getproviders.Hash{hash1, hash3} + a.SetProvider(boopProvider, v2, v2EqConstraints, hashesA) + b.SetProvider(boopProvider, v2, v2EqConstraints, hashesB) + equalProviderAddressBothWays(t, a, b) + }) +} diff --git a/website/docs/cli/commands/init.html.md b/website/docs/cli/commands/init.html.md index 21ef204706..c0c2f12b55 100644 --- a/website/docs/cli/commands/init.html.md +++ b/website/docs/cli/commands/init.html.md @@ -157,6 +157,14 @@ You can modify `terraform init`'s plugin behavior with the following options: You can use `-plugin-dir` as a one-time override for exceptional situations, such as if you are testing a local build of a provider plugin you are currently developing. +- `-lockfile=MODE` Set a dependency lockfile mode. + +The valid values for the lockfile mode are as follows: + +- readonly: suppress the lockfile changes, but verify checksums against the + information already recorded. It conflicts with the `-upgrade` flag. If you + update the lockfile with third-party dependency management tools, it would be + useful to control when it changes explicitly. ## Running `terraform init` in automation