Automatically translate dependency lock file entries when switching from OpenTofu's predecessor (#2791)

Signed-off-by: Martin Atkins <mart@degeneration.co.uk>

CHANGELOG.md

@@ -44,6 +44,7 @@ ENHANCEMENTS:
 * `removed` now supports `lifecycle` and `provisioner` configuration. ([#2556](https://github.com/opentofu/opentofu/issues/2556))
 * "force-unlock" option is now supported by the HTTP backend. ([#2381](https://github.com/opentofu/opentofu/pull/2381))
 * Module version constraints now support `null` values, which are treated as if no version was specified. ([#2660](https://github.com/opentofu/opentofu/pull/2660))
+* When running `tofu init` with a dependency lock file that contains entries for certain providers on `registry.terraform.io`, OpenTofu now attempts to select the corresponding version of the equivalent provider on `registry.opentofu.org` as an aid when switching directly from OpenTofu's predecessor. This applies only to the providers that are rebuilt from source and republished on the OpenTofu Registry by the OpenTofu project, because we cannot assume any equivalents for third-party providers published in other namespaces. ([#2791](https://github.com/opentofu/opentofu/pull/2791))
 
 BUG FIXES:
 

internal/command/e2etest/providers_tamper_test.go

@@ -8,9 +8,14 @@ package e2etest
 import (
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/opentofu/opentofu/internal/depsfile"
 	"github.com/opentofu/opentofu/internal/e2e"
 	"github.com/opentofu/opentofu/internal/getproviders"
 )
@@ -265,6 +270,99 @@ func TestProviderTampering(t *testing.T) {
 	})
 }
 
+// TestProviderLocksFromPredecessorProject is an end-to-end test of our
+// special treatment of lock files that were originally created by the
+// project that OpenTofu was forked from, and so refer to providers from
+// that project's registry instead of OpenTofu's registry.
+//
+// In that case we attempt to adjust the lock file so that we'll select
+// the same version of the equivalent provider in the OpenTofu registry,
+// even though normally OpenTofu would see the providers in two different
+// registries as completely distinct.
+//
+// This special behavior applies only to providers that match
+// registry.terraform.io/hashicorp/*, since those are the ones that the
+// OpenTofu project rebuilds and republishes with equivalent releases under
+// registry.opentofu.org/hashicorp/*.
+func TestProviderLocksFromPredecessorProject(t *testing.T) {
+	t.Parallel()
+
+	// This test reaches out to registry.opentofu.org to download the
+	// null provider, so it can only run if network access is allowed.
+	skipIfCannotAccessNetwork(t)
+
+	fixturePath := filepath.Join("testdata", "predecessor-dependency-lock-file")
+	tf := e2e.NewBinary(t, tofuBin, fixturePath)
+
+	stdout, stderr, err := tf.Run("init")
+	if err != nil {
+		t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr)
+	}
+	if !strings.Contains(stdout, "Installing hashicorp/null v3.2.0") {
+		t.Errorf("null provider download message is missing from init output:\n%s", stdout)
+		t.Logf("(if the output specifies a version other than v3.2.0 then the fixup behavior did not work correctly)")
+	}
+	if !strings.Contains(stdout, "- registry.terraform.io/hashicorp/null => registry.opentofu.org/hashicorp/null") {
+		t.Errorf("null provider dependency lock fixup message is missing from init output:\n%s", stdout)
+	}
+
+	// The lock file should have been updated to include the selection for
+	// OpenTofu-flavored version of the provider along with the checksums
+	// of OpenTofu's release, and the original entry should've been pruned
+	// because as far as OpenTofu is concerned there's no dependency on
+	// that provider in the current configuration.
+	newLocks, err := tf.ReadFile(".terraform.lock.hcl")
+	if err != nil {
+		t.Fatalf("failed to load dependency lock file after init: %s", err)
+	}
+	locks, diags := depsfile.LoadLocksFromBytes(newLocks, ".terraform.lock.hcl")
+	if diags.HasErrors() {
+		t.Fatalf("failed to load dependency lock file after init: %s", diags.Err())
+	}
+
+	if lock := locks.Provider(addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/null")); lock != nil {
+		t.Errorf("still have entry for %s v%s after init", lock.Provider(), lock.Version())
+	}
+
+	if lock := locks.Provider(addrs.MustParseProviderSourceString("registry.opentofu.org/hashicorp/null")); lock != nil {
+		if got, want := lock.Version(), getproviders.MustParseVersion("3.2.0"); got != want {
+			t.Errorf("wrong version of %s was selected\ngot:  %s\nwant: %s", lock.Provider(), got, want)
+		}
+		// The full set of hashes we captured will vary depending on the
+		// platform where this test is running, but the "zh:" ones in
+		// particular come from the remote registry rather than from local
+		// calculation and so we'll assume they should be consistent.
+		allHashes := lock.AllHashes()
+		wantHashes := []getproviders.Hash{
+			// These are the official hashes for OpenTofu's build of
+			// hashicorp/null v3.2.0, as recorded in the registry.
+			getproviders.HashSchemeZip.New("11d576a7c9b9b5c3263fae11962216e8bce9e80ab9c5c7e2635a94f410d723f0"),
+			getproviders.HashSchemeZip.New("11e53de20574d5e449c2d4e4f4249644244bad2a365e9793267796b9b96befab"),
+			getproviders.HashSchemeZip.New("1eea180daf676f35e38aa0ca237334d86bdc7a4fd78da54c139d8c6e15ad0b7e"),
+			getproviders.HashSchemeZip.New("47645b42501cb29acc270b99f93bf96bdae649159f2b3fdfafbc9543c36930e1"),
+			getproviders.HashSchemeZip.New("639854d0182d91224e67b512bcc7d12705d7aca0095b2969c65680527402eef9"),
+			getproviders.HashSchemeZip.New("894a3a5980bbe7e3d2948e0bcf56ae28b4ac16aa28c69f9a104c70af0f2f7ee1"),
+			getproviders.HashSchemeZip.New("a4b4709333738c9e14cd285879f24792d8a2e277f071c9c641b11e5289c854f3"),
+			getproviders.HashSchemeZip.New("c0fa29f9e93525f4672ea91b61ed866624ba3f3afd64d1c9eff8cc4c319ba69b"),
+			getproviders.HashSchemeZip.New("f77678a6b62eb332d867cb7671982100f463d20a0f115c88a5d23f516ee872fa"),
+			getproviders.HashSchemeZip.New("f7a8ab5f6b6c54667c240c8d8ed9c45a46bdbfa6bead009198a30def88e35376"),
+		}
+		var gotHashes []getproviders.Hash
+		for _, hash := range allHashes {
+			if hash.HasScheme(getproviders.HashSchemeZip) {
+				gotHashes = append(gotHashes, hash)
+			}
+		}
+		slices.Sort(gotHashes) // order is unimportant
+		if diff := cmp.Diff(wantHashes, gotHashes); diff != "" {
+			t.Error("wrong hashes in lock file after init\n" + diff)
+		}
+	} else {
+		t.Errorf("missing entry for registry.opentofu.org/hashicorp/null after init")
+	}
+
+}
+
 const localBackendConfig = `
 terraform {
   backend "local" {

internal/command/e2etest/testdata/predecessor-dependency-lock-file/.terraform.lock.hcl

@@ -0,0 +1,23 @@
+
+# This intentionally refers to the registry of OpenTofu's predecessor, but
+# the associated configuration refers to the shorthand "hashicorp/null"
+# and so will be understood by OpenTofu as depending instead on
+# "registry.opentofu.org/hashicorp/null", thereby activating our special
+# fixup behavior and selecting the same version of OpenTofu's re-release
+# of this provider.
+provider "registry.terraform.io/hashicorp/null" {
+  version = "3.2.0"
+  hashes = [
+    "h1:uQv2oPjJv+ue8bPrVp+So2hHd90UTssnCNajTW554Cw=",
+    "zh:40335019c11e5bdb3780301da5197cbc756b4b5ac3d699c52583f1d34e963176",
+    "zh:42356e687656fc8ec1f230f786f830f344e64419552ec483e2bc79bd4b7cf1e8",
+    "zh:5ce03460813954cbebc9f9ad5befbe364d9dc67acb08869f67c1aa634fbf6d6c",
+    "zh:658ea3e3e7ecc964bdbd08ecde63f3d79f298bab9922b29a6526ba941a4d403a",
+    "zh:68c06703bc57f9c882bfedda6f3047775f0d367093d00efb040800c798b8a613",
+    "zh:80fd03335f793dc54302dd53da98c91fd94f182bcacf13457bed1a99ecffbc1a",
+    "zh:91a76f371815a130735c8fcb6196804d878aebcc67b4c3b73571d2063336ffd8",
+    "zh:c146fc0291b7f6284fe4d54ce6aaece6957e9acc93fc572dd505dfd8bcad3e6c",
+    "zh:c38c9a295cfae9fb6372523c34b9466cd439d5e2c909b56a788960d387c24320",
+    "zh:e25265d4e87821d18dc9653a0ce01978a1ae5d363bc01dd273454db1aa0309c7",
+  ]
+}

internal/command/e2etest/testdata/predecessor-dependency-lock-file/predecessor-dependency-lock-file.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    null = {
+      # Since this source address doesn't include an explicit hostname,
+      # it gets treated as a shorthand for the default registry, which
+      # differs between OpenTofu and its predecessor.
+      source = "hashicorp/null"
+    }
+  }
+}

internal/command/meta_dependencies.go

@@ -7,9 +7,14 @@ package command
 
 import (
 	"context"
+	"fmt"
 	"log"
+	"maps"
 	"os"
+	"slices"
+	"strings"
 
+	"github.com/opentofu/opentofu/internal/addrs"
 	"github.com/opentofu/opentofu/internal/depsfile"
 	"github.com/opentofu/opentofu/internal/tfdiags"
 )
@@ -59,6 +64,39 @@ func (m *Meta) lockedDependencies() (*depsfile.Locks, tfdiags.Diagnostics) {
 	}
 
 	ret, diags := depsfile.LoadLocksFromFile(dependencyLockFilename)
+
+	// If this is the first run after switching from OpenTofu's predecessor,
+	// the lock file might contain some entries from the predecessor's registry
+	// which we can translate into similar entries for OpenTofu's registry.
+	changed := ret.UpgradeFromPredecessorProject()
+	if len(changed) != 0 {
+		oldAddrs := slices.Collect(maps.Keys(changed))
+		slices.SortFunc(oldAddrs, func(a, b addrs.Provider) int {
+			if a.LessThan(b) {
+				return -1
+			} else if b.LessThan(a) {
+				return 1
+			} else {
+				return 0
+			}
+		})
+		var buf strings.Builder // strings.Builder writes cannot fail
+		_, _ = buf.WriteString("OpenTofu automatically rewrote some entries in your dependency lock file:\n")
+		for _, oldAddr := range oldAddrs {
+			newAddr := changed[oldAddr]
+			// We intentionally use String instead of ForDisplay here because
+			// this message won't make much sense without using fully-qualified
+			// addresses with explicit registry hostnames.
+			_, _ = fmt.Fprintf(&buf, "  - %s => %s\n", oldAddr.String(), newAddr.String())
+		}
+		_, _ = buf.WriteString("\nThe version selections were preserved, but the hashes were not because the OpenTofu project's provider releases are not byte-for-byte identical.")
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Warning,
+			"Dependency lock file entries automatically updated",
+			buf.String(),
+		))
+	}
+
 	return m.annotateDependencyLocksWithOverrides(ret), diags
 }
 

internal/depsfile/locks.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"sort"
 
+	svchost "github.com/hashicorp/terraform-svchost"
+
 	"github.com/opentofu/opentofu/internal/addrs"
 	"github.com/opentofu/opentofu/internal/getproviders"
 )
@@ -298,6 +300,59 @@ func (l *Locks) EqualProviderAddress(other *Locks) bool {
 	return true
 }
 
+// UpgradeFromPredecessorProject checks the stored locks for any that seem
+// like they were probably created by the project that OpenTofu forked from,
+// and if so adds similar lock entries for same-named providers in the OpenTofu
+// registry.
+//
+// Because there is no guarantee that hashes would match between providers in
+// the two registries, the replacement lock file entries initially have no
+// hashes at all, and so can be used to select the same version number but
+// not to guarantee that the provider package matches what was previously
+// installed. The lock entries for the "old" providers are retained by this
+// function, but are likely to be discarded if the locks are subsequently used
+// for provider installation, since that discards lock entries for any providers
+// not currently in use.
+//
+// The result is a map from the providers whose entries were replaced to the
+// new provider that was selected to replace each one, so that the caller can
+// explain the changes to the operator. The result is zero-length if no
+// changes were made.
+func (l *Locks) UpgradeFromPredecessorProject() map[addrs.Provider]addrs.Provider {
+	var ret map[addrs.Provider]addrs.Provider // initialized only if we change something
+
+	for oldAddr, oldLock := range l.providers {
+		if oldAddr.Hostname != svchost.Hostname("registry.terraform.io") || oldAddr.Namespace != "hashicorp" {
+			// Only providers in this namespace have corresponding providers
+			// in the OpenTofu registry that are controlled by the OpenTofu
+			// project. We cannot safely make any assumption of equivalence
+			// about providers in any other registry or namespace.
+			continue
+		}
+		newAddr := addrs.NewDefaultProvider(oldAddr.Type)
+		if _, ok := l.providers[newAddr]; ok {
+			continue // we already have an entry for the replacement provider, so we'll keep it
+		}
+
+		// Our new lock file entry retains the same version selection and
+		// version constraints but discards the hashes.
+		newLock := NewProviderLock(newAddr, oldLock.Version(), oldLock.VersionConstraints(), nil)
+		l.providers[newAddr] = newLock
+		// NOTE: We intentionally keep the old entry in here for now, and
+		// let subsequent provider installer work clean it up, because the
+		// provider installer can tell whether anything in the configuration
+		// or state is somehow still referring to the old provider, in which
+		// case we ought to keep the entry for it.
+
+		if ret == nil {
+			ret = make(map[addrs.Provider]addrs.Provider)
+		}
+		ret[oldAddr] = newAddr
+	}
+
+	return ret
+}
+
 // Empty returns true if the given Locks object contains no actual locks.
 //
 // UI code might wish to use this to distinguish a lock file being

internal/depsfile/locks_test.go

@@ -312,3 +312,105 @@ func TestProviderLockContainsAll(t *testing.T) {
 		}
 	})
 }
+
+func TestLocksUpgradeFromPredecessorProject(t *testing.T) {
+	locks, diags := LoadLocksFromBytes([]byte(`
+		provider "registry.terraform.io/hashicorp/a" {
+			version = "1.0.0"
+			constraints = ">= 1.0.0"
+			hashes = [
+				"h1:jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0=",
+			]
+		}
+
+		provider "registry.terraform.io/anything-else/b" {
+			version = "2.0.0"
+			hashes = [
+				"h1:jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0=",
+			]
+		}
+
+		provider "registry.terraform.io/hashicorp/c" {
+			version = "3.0.0"
+			hashes = [
+				"h1:jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0=",
+			]
+		}
+		provider "registry.opentofu.org/hashicorp/c" {
+			version = "4.0.0"
+			hashes = [
+				"h1:jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9T0Mh=",
+			]
+		}
+	`), "")
+	if diags.HasErrors() {
+		t.Fatalf("unexpected errors: %s", diags.Err().Error())
+	}
+
+	gotChanges := locks.UpgradeFromPredecessorProject()
+	wantChanges := map[addrs.Provider]addrs.Provider{
+		addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/a"): addrs.MustParseProviderSourceString("registry.opentofu.org/hashicorp/a"),
+	}
+	if diff := cmp.Diff(wantChanges, gotChanges); diff != "" {
+		t.Error("wrong reported changes\n" + diff)
+	}
+
+	gotLocks := locks.AllProviders()
+	wantLocks := map[addrs.Provider]*ProviderLock{
+		// This one is still included because we want to let the provider
+		// installer be the one to decide to remove it, once it's convinced
+		// itself that this provider is definitely no longer needed...
+		addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/a"): {
+			addr:               addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/a"),
+			version:            getproviders.MustParseVersion("1.0.0"),
+			versionConstraints: getproviders.MustParseVersionConstraints(">= 1.0.0"),
+			hashes: []getproviders.Hash{
+				getproviders.HashScheme1.New("jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0="),
+			},
+		},
+
+		// ...but we now have this new one that describes the OpenTofu equivalent
+		// of it, with the same version but not yet any hashes. The hashes
+		// must be determined by a subsequent installation step.
+		addrs.MustParseProviderSourceString("registry.opentofu.org/hashicorp/a"): {
+			addr:               addrs.MustParseProviderSourceString("registry.opentofu.org/hashicorp/a"),
+			version:            getproviders.MustParseVersion("1.0.0"),
+			versionConstraints: getproviders.MustParseVersionConstraints(">= 1.0.0"),
+			// intentionally no hashes here, but the version and versionConstraints
+			// must match the entry above.
+		},
+
+		// This one does not get any special treatment because it's not in
+		// the namespace where the OpenTofu project maintains
+		// directly-corresponding releases.
+		addrs.MustParseProviderSourceString("registry.terraform.io/anything-else/b"): {
+			addr:    addrs.MustParseProviderSourceString("registry.terraform.io/anything-else/b"),
+			version: getproviders.MustParseVersion("2.0.0"),
+			hashes: []getproviders.Hash{
+				getproviders.HashScheme1.New("jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0="),
+			},
+		},
+
+		// The following two both survive unchanged because we don't want to
+		// destroy any existing lock file entry for a provider from the
+		// OpenTofu registry even if there's an entry that could potentially
+		// have been translated.
+		addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/c"): {
+			addr:    addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/c"),
+			version: getproviders.MustParseVersion("3.0.0"),
+			hashes: []getproviders.Hash{
+				getproviders.HashScheme1.New("jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9MhT0="),
+			},
+		},
+		addrs.MustParseProviderSourceString("registry.opentofu.org/hashicorp/c"): {
+			addr:    addrs.MustParseProviderSourceString("registry.opentofu.org/hashicorp/c"),
+			version: getproviders.MustParseVersion("4.0.0"),
+			hashes: []getproviders.Hash{
+				getproviders.HashScheme1.New("jsKjBiLb+v3OIC3xuDiY4sR0r1OHUMSWPYKult9T0Mh="),
+			},
+		},
+	}
+	if diff := cmp.Diff(wantLocks, gotLocks, ProviderLockComparer); diff != "" {
+		t.Error("wrong updated locks\n" + diff)
+	}
+}