diff --git a/.changes/v1.16/BUG FIXES-20260522-180848.yaml b/.changes/v1.16/BUG FIXES-20260522-180848.yaml new file mode 100644 index 0000000000..c457ee40a2 --- /dev/null +++ b/.changes/v1.16/BUG FIXES-20260522-180848.yaml @@ -0,0 +1,5 @@ +kind: BUG FIXES +body: 'init: Stop removing locks from the dependency lock file corresponding to providers configured as a dev_override' +time: 2026-05-22T18:08:48.823127+01:00 +custom: + Issue: "38634" diff --git a/.changes/v1.16/NOTES-20260526-164738.yaml b/.changes/v1.16/NOTES-20260526-164738.yaml new file mode 100644 index 0000000000..9e59c889c2 --- /dev/null +++ b/.changes/v1.16/NOTES-20260526-164738.yaml @@ -0,0 +1,5 @@ +kind: NOTES +body: 'providers: The `providers locks` command will now warn users about any dev_overrides in effect, as these will stop provider locks from being downloaded.' +time: 2026-05-26T16:47:38.947615+01:00 +custom: + Issue: "38634" diff --git a/internal/command/e2etest/pluggable_state_store_test.go b/internal/command/e2etest/pluggable_state_store_test.go index f9c389e917..73cef31ba2 100644 --- a/internal/command/e2etest/pluggable_state_store_test.go +++ b/internal/command/e2etest/pluggable_state_store_test.go @@ -23,9 +23,9 @@ import ( ) // Test that users can do the full init-plan-apply workflow with pluggable state storage -// when the state storage provider is reattached/unmanaged by Terraform. +// when the state storage provider is unmanaged by Terraform. // As well as ensuring that the state store can be initialised ok, this tests that -// the state store's details can be stored in the plan file despite the fact it's reattached. +// the state store's details can be stored in the plan file despite the fact it's unmanaged. func TestPrimary_stateStore_unmanaged_separatePlan(t *testing.T) { if !canRunGoBuild { // We're running in a separate-build-then-run context, so we can't diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index b475e9ffb5..66f0f58eb0 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -396,9 +396,9 @@ func TestPrimary_stateStore_swapProviderSupplyMode_betweenInitAndPlanApply(t *te // that change doesn't impact the hash of the state store. The hash is impacted by the Version data, and all unmanaged // providers used for PSS will have null version data. // - // In contrast, swapping between a managed provider and any of reattached/dev_override/builtin WILL trigger a hash mismatch + // In contrast, swapping between a managed provider and any of unmanaged/dev_override/builtin WILL trigger a hash mismatch // because the version data will change. - t.Run("users are NOT prompted to migrate state if an unmanaged provider used for PSS provider swaps supply mode (e.g. swap from reattached to dev_override) between init and plan+apply", func(t *testing.T) { + t.Run("users are NOT prompted to migrate state if an unmanaged provider used for PSS provider swaps supply mode (e.g. swap from unmanaged to dev_override) between init and plan+apply", func(t *testing.T) { if !canRunGoBuild { // We're running in a separate-build-then-run context, so we can't // currently execute this test which depends on being able to build @@ -417,13 +417,13 @@ func TestPrimary_stateStore_swapProviderSupplyMode_betweenInitAndPlanApply(t *te reattachStr, _ := reattachedProviderForTest(t, addrs.NewDefaultProvider("simple6"), 6) tf.AddEnv("TF_REATTACH_PROVIDERS=" + string(reattachStr)) - //// INIT - using reattached provider. + //// INIT - using unmanaged provider. _, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-no-color") if err != nil { t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) } - // Assert backend state file says the provider is a reattached + // Assert backend state file says the provider is unmanaged statePath := filepath.Join(tf.WorkDir(), ".terraform", command.DefaultStateFilename) sMgr := &clistate.LocalState{Path: statePath} if err := sMgr.RefreshState(); err != nil { @@ -439,7 +439,7 @@ func TestPrimary_stateStore_swapProviderSupplyMode_betweenInitAndPlanApply(t *te //// PLAN - using same provider but supplied via dev_override instead of reattach config. - // No longer using reattached providers. + // No longer using unmanaged providers. tf.RemoveEnv("TF_REATTACH_PROVIDERS") // Build the provider binary and direct Terraform to use it via dev_override, which should cause Terraform to treat it as a dev_override in a CLI configuration file. @@ -683,9 +683,9 @@ func TestPrimary_stateStore_swapProviderSupplyMode_betweenSuccessiveInits(t *tes // that change doesn't impact the hash of the state store. The hash is impacted by the Version data, and all unmanaged // providers used for PSS will have null version data. // - // In contrast, swapping between a managed provider and any of reattached/dev_override/builtin WILL trigger a hash mismatch + // In contrast, swapping between a managed provider and any of unmanaged/dev_override/builtin WILL trigger a hash mismatch // because the version data will change. - t.Run("users are NOT prompted to migrate state if an unmanaged provider used for PSS provider swaps supply mode (e.g. swap from reattached to dev_override) between init and plan+apply", func(t *testing.T) { + t.Run("users are NOT prompted to migrate state if an unmanaged provider used for PSS provider swaps supply mode (e.g. swap from unmanaged to dev_override) between init and plan+apply", func(t *testing.T) { if !canRunGoBuild { // We're running in a separate-build-then-run context, so we can't // currently execute this test which depends on being able to build @@ -704,13 +704,13 @@ func TestPrimary_stateStore_swapProviderSupplyMode_betweenSuccessiveInits(t *tes reattachStr, _ := reattachedProviderForTest(t, addrs.NewDefaultProvider("simple6"), 6) tf.AddEnv("TF_REATTACH_PROVIDERS=" + string(reattachStr)) - //// INIT 1 - using reattached provider. + //// INIT 1 - using unmanaged provider. _, stderr, err := tf.Run("init", "-enable-pluggable-state-storage-experiment=true", "-no-color") if err != nil { t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) } - // Assert backend state file says the provider is a reattached + // Assert backend state file says the provider is unmanaged statePath := filepath.Join(tf.WorkDir(), ".terraform", command.DefaultStateFilename) sMgr := &clistate.LocalState{Path: statePath} if err := sMgr.RefreshState(); err != nil { @@ -726,7 +726,7 @@ func TestPrimary_stateStore_swapProviderSupplyMode_betweenSuccessiveInits(t *tes //// INIT 2 - using same provider but supplied via dev_override instead of reattach config. - // No longer using reattached providers. + // No longer using unmanaged providers. tf.RemoveEnv("TF_REATTACH_PROVIDERS") // Build the provider binary and direct Terraform to use it via dev_override, which should cause Terraform to treat it as a dev_override in a CLI configuration file. diff --git a/internal/command/e2etest/provider_plugin_test.go b/internal/command/e2etest/provider_plugin_test.go index 44fe5b1d3f..513804d5d0 100644 --- a/internal/command/e2etest/provider_plugin_test.go +++ b/internal/command/e2etest/provider_plugin_test.go @@ -248,7 +248,7 @@ provider "registry.terraform.io/hashicorp/simple" { } }) - t.Run("dev_override causes provider to be removed from dependency lock file during init", func(t *testing.T) { + t.Run("dev_override providers are still represented in the dependency lock file after init", func(t *testing.T) { terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") tf := e2e.NewBinary(t, terraformBin, fixturePath) @@ -286,29 +286,18 @@ provider "registry.terraform.io/hashicorp/simple6" { t.Fatalf("unexpected error: %s\nstdout: %s\nstderr: %s", err, stdout, stderr) } - // Lockfile has been altered to remove the simple6 provider + // Lockfile is unchanged despite use of a dev_override simple6 provider buf, err := os.ReadFile(lockFile) if err != nil { t.Fatalf("unexpected error accessing lock file: %s", err) } buf = bytes.TrimSpace(buf) - expectedLockFile := fmt.Sprintf(`# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/simple" { - version = "1.0.0" - hashes = [ - "%s", - ] -}`, - simple5v1_0_0Hash, - ) - if diff := cmp.Diff(expectedLockFile, string(buf)); diff != "" { + if diff := cmp.Diff(priorLockFile, string(buf)); diff != "" { t.Fatalf("unexpected difference in lock file content: %s", diff) } }) - t.Run("dev_override also causes provider to be removed from dependency lock file during init -upgrade", func(t *testing.T) { + t.Run("dev_override providers are unchanged in the dependency lock file during init -upgrade", func(t *testing.T) { terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") tf := e2e.NewBinary(t, terraformBin, fixturePath) @@ -363,8 +352,16 @@ provider "registry.terraform.io/hashicorp/simple" { hashes = [ "%s", ] +} + +provider "registry.terraform.io/hashicorp/simple6" { + version = "1.0.0" + hashes = [ + "%s", + ] }`, simple5v2_0_0Hash, + simple6v1_0_0Hash, // not upgraded to 2.0.0 ) if diff := cmp.Diff(expectedLockFileContent, string(buf)); diff != "" { t.Errorf("unexpected difference in lock file content: %s", diff) @@ -372,9 +369,9 @@ provider "registry.terraform.io/hashicorp/simple" { }) } -// TestProviderInstall_reattached verifies provider plugin installation behaviour -// when a reattached/unmanaged provider is in use. -func TestProviderInstall_reattached(t *testing.T) { +// TestProviderInstall_unmanaged verifies provider plugin installation behaviour +// when an unmanaged provider is in use. +func TestProviderInstall_unmanaged(t *testing.T) { if !canRunGoBuild { // We're running in a separate-build-then-run context, so we can't // currently execute this test which depends on being able to build @@ -448,11 +445,11 @@ func TestProviderInstall_reattached(t *testing.T) { t.Fatal(err) } - // Launch a separate simple6 provider process to be re-used as a reattached provider. + // Launch a separate simple6 provider process to be re-used as an unmanaged provider. // Tests will use this via the TF_REATTACH_PROVIDERS environment variable. reattachConfig, _ := reattachedProviderForTest(t, addrs.NewDefaultProvider("simple6"), 6) - t.Run("reattached provider not installed when provider not present in dependency lock file", func(t *testing.T) { + t.Run("unmanaged provider not installed when provider not present in dependency lock file", func(t *testing.T) { terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") tf := e2e.NewBinary(t, terraformBin, fixturePath) @@ -466,7 +463,7 @@ func TestProviderInstall_reattached(t *testing.T) { t.Fatalf("expected error due to file not existing, got different error: %s", err) } - // The simple6 provider is reattached/unmanaged + // The simple6 provider is unmanaged tf.AddEnv("TF_REATTACH_PROVIDERS=" + reattachConfig) // The init process should succeed. @@ -482,7 +479,7 @@ func TestProviderInstall_reattached(t *testing.T) { } buf = bytes.TrimSpace(buf) - // We expect the lock file to not contain the simple6 provider that's being reattached/unmanaged, + // We expect the lock file to not contain the simple6 provider that's being unmanaged, // because that provider is skipped during the installation process. // The simple (v5) provider is installed as usual, pulling in the latest version. expectedLockFileContent := fmt.Sprintf(`# This file is maintained automatically by "terraform init". @@ -500,7 +497,7 @@ provider "registry.terraform.io/hashicorp/simple" { } }) - t.Run("reattached providers do NOT cause provider to be removed from dependency lock file during init", func(t *testing.T) { + t.Run("unmanaged providers are still represented in the dependency lock file after init", func(t *testing.T) { terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") tf := e2e.NewBinary(t, terraformBin, fixturePath) @@ -529,7 +526,7 @@ provider "registry.terraform.io/hashicorp/simple6" { t.Fatalf("error writing prior lock file: %s", err) } - // The simple6 provider is reattached/unmanaged + // The simple6 provider is unmanaged tf.AddEnv("TF_REATTACH_PROVIDERS=" + reattachConfig) // The init process should succeed. @@ -538,7 +535,7 @@ provider "registry.terraform.io/hashicorp/simple6" { t.Fatalf("unexpected error: %s\nstdout: %s\nstderr: %s", err, stdout, stderr) } - // Lockfile is unchanged despite use of a reattached/unmanaged simple6 provider + // Lockfile is unchanged despite use of an unmanaged simple6 provider buf, err := os.ReadFile(lockFile) if err != nil { t.Fatalf("unexpected error accessing lock file: %s", err) @@ -549,7 +546,7 @@ provider "registry.terraform.io/hashicorp/simple6" { } }) - t.Run("reattached providers are unchanged in the dependency lock file during init -upgrade", func(t *testing.T) { + t.Run("unmanaged providers are unchanged in the dependency lock file during init -upgrade", func(t *testing.T) { terraformBin := e2e.GoBuild("github.com/hashicorp/terraform", "terraform") tf := e2e.NewBinary(t, terraformBin, fixturePath) @@ -578,7 +575,7 @@ provider "registry.terraform.io/hashicorp/simple6" { t.Fatalf("error writing prior lock file: %s", err) } - // The simple6 provider is reattached/unmanaged + // The simple6 provider is unmanaged tf.AddEnv("TF_REATTACH_PROVIDERS=" + reattachConfig) // The init -upgrade process should succeed. @@ -589,7 +586,7 @@ provider "registry.terraform.io/hashicorp/simple6" { // Lockfile shows evidence of upgrade process // simple provider is upgraded to the newer 2.0.0 version, - // but the reattached simple6 provider is unchanged due to being reattached. + // but the unmanaged simple6 provider is unchanged due to being unmanaged. buf, err := os.ReadFile(lockFile) if err != nil { t.Fatalf("unexpected error accessing lock file: %s", err) @@ -613,7 +610,7 @@ provider "registry.terraform.io/hashicorp/simple6" { ] }`, simple5v2_0_0Hash, - simple6v1_0_0Hash, + simple6v1_0_0Hash, // not upgraded to 2.0.0 ) if diff := cmp.Diff(expectedLockFileContent, string(buf)); diff != "" { t.Errorf("unexpected difference in lock file content: %s", diff) @@ -621,7 +618,7 @@ provider "registry.terraform.io/hashicorp/simple6" { }) } -// reattachedProviderForTest launches a provider process and returns a reattach config string +// reattachConfigForTest launches a provider process and returns a reattach config string // that can be used as the value for the TF_REATTACH_PROVIDERS environment variable in tests. // Cleanup of the provider process is handled internally. func reattachedProviderForTest(t *testing.T, provider addrs.Provider, protocol int) (string, *providerServer) { diff --git a/internal/command/init.go b/internal/command/init.go index fdfe291cf3..e8b3885af3 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -401,8 +401,6 @@ func (c *InitCommand) getProvidersFromConfig(ctx context.Context, config *config return false, nil, SafeInitActionInvalid, nil, diags } - reqs = c.removeDevOverrides(reqs) - for providerAddr := range reqs { if providerAddr.IsLegacy() { diags = diags.Append(tfdiags.Sourceless( diff --git a/internal/command/meta_providers.go b/internal/command/meta_providers.go index d194115d54..66621156ce 100644 --- a/internal/command/meta_providers.go +++ b/internal/command/meta_providers.go @@ -18,7 +18,6 @@ import ( builtinProviders "github.com/hashicorp/terraform/internal/builtin/providers" "github.com/hashicorp/terraform/internal/depsfile" "github.com/hashicorp/terraform/internal/getproviders" - "github.com/hashicorp/terraform/internal/getproviders/providerreqs" "github.com/hashicorp/terraform/internal/logging" tfplugin "github.com/hashicorp/terraform/internal/plugin" tfplugin6 "github.com/hashicorp/terraform/internal/plugin6" @@ -74,11 +73,21 @@ func (m *Meta) providerInstallerCustomSource(source getproviders.Source) *provid builtinProviderTypes = append(builtinProviderTypes, ty) } inst.SetBuiltInProviderTypes(builtinProviderTypes) + + // Overridden providers consist of both: + // 1. reattached providers + // 2. development override providers unmanagedProviderTypes := make(map[addrs.Provider]struct{}, len(m.UnmanagedProviders)) for ty := range m.UnmanagedProviders { unmanagedProviderTypes[ty] = struct{}{} } inst.SetUnmanagedProviderTypes(unmanagedProviderTypes) + devOverrideProviderTypes := make(map[addrs.Provider]struct{}, len(m.ProviderDevOverrides)) + for ty := range m.ProviderDevOverrides { + devOverrideProviderTypes[ty] = struct{}{} + } + inst.SetDevOverrideTypes(devOverrideProviderTypes) + return inst } @@ -187,6 +196,29 @@ func (m *Meta) providerDevOverrideInitWarnings() tfdiags.Diagnostics { } } +// providerDevOverrideProviderLockWarnings is just like providerDevOverrideInitWarnings +// except the diagnostic is written with a message specific to the `providers lock` command. +// Similarly, diags will only be returned if there is 1+ dev_overrides in effect, and no error +// diags will be returned. +func (m *Meta) providerDevOverrideProvidersLockWarnings() tfdiags.Diagnostics { + if len(m.ProviderDevOverrides) == 0 { + return nil + } + var detailMsg strings.Builder + detailMsg.WriteString("The following provider development overrides are set in the CLI configuration:\n") + for addr, path := range m.ProviderDevOverrides { + detailMsg.WriteString(fmt.Sprintf(" - %s in %s\n", addr.ForDisplay(), path)) + } + detailMsg.WriteString("\nThese provider locks will not be recorded because the provider is overwritten. If this is unintentional please re-run without the development overrides set.") + return tfdiags.Diagnostics{ + tfdiags.Sourceless( + tfdiags.Warning, + "Provider development overrides are in effect", + detailMsg.String(), + ), + } +} + func (m *Meta) isProviderDevOverride(pAddr addrs.Provider) bool { if len(m.ProviderDevOverrides) == 0 { return false @@ -195,21 +227,6 @@ func (m *Meta) isProviderDevOverride(pAddr addrs.Provider) bool { return overridden } -func (m *Meta) removeDevOverrides(reqs providerreqs.Requirements) providerreqs.Requirements { - // Deep copy the requirements to avoid mutating the input - copiedReqs := make(providerreqs.Requirements) - for provider, versions := range reqs { - // Only copy if the provider is not overridden - if _, overridden := m.ProviderDevOverrides[provider]; !overridden { - copiedVersions := make(providerreqs.VersionConstraints, len(versions)) - copy(copiedVersions, versions) - copiedReqs[provider] = copiedVersions - } - } - - return copiedReqs -} - // providerDevOverrideRuntimeWarnings returns a diagnostics that contains at // least one warning if and only if there is at least one provider development // override in effect. If not, the result is always empty. The result never diff --git a/internal/command/meta_providers_test.go b/internal/command/meta_providers_test.go index de04620a07..0b50488985 100644 --- a/internal/command/meta_providers_test.go +++ b/internal/command/meta_providers_test.go @@ -144,7 +144,7 @@ func TestEnsureProviderVersions_devOverrideAndReattachedProviders(t *testing.T) providerA.ForDisplay(), providerB.ForDisplay(), providerC.ForDisplay(), - providerD.ForDisplay(), // D is installed despite being dev overridden + // D is not installed due to being dev override }, }, diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index 4430565dd0..09cde2f3ba 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -195,6 +195,11 @@ func (c *ProvidersLockCommand) Run(args []string) int { // merge all of the generated locks together at the end. updatedLocks := map[getproviders.Platform]*depsfile.Locks{} selectedVersions := map[addrs.Provider]getproviders.Version{} + + // Warn users about any development overrides in effect; they will block + // locks being obtained for the overridden providers. + c.showDiagnostics(c.providerDevOverrideProvidersLockWarnings()) + for _, platform := range platforms { tempDir, err := ioutil.TempDir("", "terraform-providers-lock") if err != nil { diff --git a/internal/providercache/installer.go b/internal/providercache/installer.go index d375e90ca5..936430ca53 100644 --- a/internal/providercache/installer.go +++ b/internal/providercache/installer.go @@ -58,6 +58,12 @@ type Installer struct { // lifecycle for, and therefore does not need to worry about the // installation of. unmanagedProviderTypes map[addrs.Provider]struct{} + + // devOverrideTypes is a set of provider addresses that should be + // considered implemented. Binaries of these providers are supplied + // from the users machine via CLI configuration, so Terraform does + // not need to worry about installing them. + devOverrideTypes map[addrs.Provider]struct{} } // NewInstaller constructs and returns a new installer with the given target @@ -161,6 +167,15 @@ func (i *Installer) SetUnmanagedProviderTypes(types map[addrs.Provider]struct{}) i.unmanagedProviderTypes = types } +// SetDevOverrideTypes tells the receiver to consider the providers +// indicated by the passed addrs.Providers as dev overrides. Terraform should not +// try to install these providers and record their versions in the dependency lock +// file; the binaries supplied via CLI configuration have no version information +// available. +func (i *Installer) SetDevOverrideTypes(types map[addrs.Provider]struct{}) { + i.devOverrideTypes = types +} + // EnsureProviderVersions compares the given provider requirements with what // is already available in the installer's target directory and then takes // appropriate installation actions to ensure that suitable packages @@ -237,6 +252,10 @@ func (i *Installer) EnsureProviderVersions(ctx context.Context, locks *depsfile. // unmanaged providers do not require installation continue } + if _, ok := i.devOverrideTypes[provider]; ok { + // development override providers do not require installation + continue + } acceptableVersions := versions.MeetingConstraints(versionConstraints) if !mode.forceQueryAllProviders() { // If we're not forcing potential changes of version then an