changes for pulling binary from releases.hashicorp.com

packer/plugin-getter/github/getter.go

@@ -7,10 +7,13 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -31,12 +34,22 @@ const (
 )
 
 type Getter struct {
-	Client    *github.Client
-	UserAgent string
+	Client     *github.Client
+	HttpClient *http.Client
+	UserAgent  string
 }
 
 var _ plugingetter.Getter = &Getter{}
 
+type PluginMetadata struct {
+	Versions map[string]PluginVersion `json:"versions"`
+}
+
+type PluginVersion struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
 func transformChecksumStream() func(in io.ReadCloser) (io.ReadCloser, error) {
 	return func(in io.ReadCloser) (io.ReadCloser, error) {
 		defer in.Close()
@@ -107,6 +120,35 @@ func transformVersionStream(in io.ReadCloser) (io.ReadCloser, error) {
 	return io.NopCloser(buf), nil
 }
 
+// transformReleasesVersionStream get a stream from github tags and transforms it into
+// something Packer wants, namely a json list of Release.
+func transformReleasesVersionStream(in io.ReadCloser) (io.ReadCloser, error) {
+	if in == nil {
+		return nil, fmt.Errorf("transformReleasesVersionStream got nil body")
+	}
+	defer in.Close()
+	dec := json.NewDecoder(in)
+
+	var m PluginMetadata
+	if err := dec.Decode(&m); err != nil {
+		return nil, err
+	}
+
+	var out []plugingetter.Release
+	for _, m := range m.Versions {
+		out = append(out, plugingetter.Release{
+			Version: "v" + m.Version,
+		})
+	}
+
+	buf := &bytes.Buffer{}
+	if err := json.NewEncoder(buf).Encode(out); err != nil {
+		return nil, err
+	}
+
+	return io.NopCloser(buf), nil
+}
+
 // HostSpecificTokenAuthTransport makes sure the http roundtripper only sets an
 // auth token for requests aimed at a specific host.
 //
@@ -188,6 +230,10 @@ func (gp GithubPlugin) RealRelativePath() string {
 	)
 }
 
+func (gp GithubPlugin) PluginType() string {
+	return fmt.Sprintf("packer-plugin-%s", gp.Type)
+}
+
 func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, error) {
 	ghURI, err := NewGithubPlugin(opts.PluginRequirement.Identifier)
 	if err != nil {
@@ -277,3 +323,89 @@ func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser,
 
 	return transform(resp.Body)
 }
+
+func (g *Getter) GetOfficialRelease(what string, opts plugingetter.GetOptions) (io.ReadCloser, error) {
+	ghURI, err := NewGithubPlugin(opts.PluginRequirement.Identifier)
+	if err != nil {
+		return nil, err
+	}
+
+	if g.HttpClient == nil {
+		caCert, err := ioutil.ReadFile("/Users/anshulsharma/Documents/test_official/cert.pem")
+		if err != nil {
+			panic(err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+
+		// Create custom TLS config
+		tlsConfig := &tls.Config{
+			InsecureSkipVerify: true,
+		}
+
+		g.HttpClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: tlsConfig,
+			},
+		}
+		//g.HttpClient = &http.Client{}
+	}
+
+	var req *http.Request
+	transform := func(in io.ReadCloser) (io.ReadCloser, error) {
+		return in, nil
+	}
+
+	switch what {
+	case "releases":
+		url := filepath.ToSlash("https://releases.hashicorp.com/" + ghURI.PluginType() + "/index.json")
+		req, err = http.NewRequest("GET", url, nil)
+		transform = transformReleasesVersionStream
+	case "sha256":
+		// something like https://github.com/sylviamoss/packer-plugin-comment/releases/download/v0.2.11/packer-plugin-comment_v0.2.11_x5_SHA256SUMS
+		url := filepath.ToSlash("https://releases.hashicorp.com/" + ghURI.PluginType() + "/" + opts.VersionString() + "/" + ghURI.PluginType() + "_" + opts.VersionString() + "_SHA256SUMS")
+		transform = transformChecksumStream()
+		log.Printf("[DEBUG] github-getter: getting %q", url)
+		req, err = http.NewRequest("GET", url, nil)
+	case "zip":
+		// https://releases.hashicorp.com/terraform-provider-akamai/8.0.0/terraform-provider-akamai_8.0.0_darwin_arm64.zip
+		url := filepath.ToSlash("https://releases.hashicorp.com/" + ghURI.PluginType() + "/" + opts.VersionString() + "/" + opts.ExpectedZipFilename())
+		req, err = http.NewRequest("GET", url, nil)
+		transform = transformZipStream()
+	default:
+		return nil, fmt.Errorf("%q not implemented", what)
+	}
+
+	if err != nil {
+		log.Printf("[ERROR] http-getter: error creating request for %q: %s", what, err)
+		return nil, err
+	}
+
+	resp, err := g.HttpClient.Do(req)
+	if err != nil || resp.StatusCode >= 400 {
+		log.Printf("[ERROR] Got error while getting data from releases.hashicorp.com, %v", err)
+		return nil, plugingetter.HTTPFailure
+	}
+
+	defer func(Body io.ReadCloser) {
+		err = Body.Close()
+		if err != nil {
+			log.Printf("[ERROR] http-getter: error closing response body: %s", err)
+		}
+	}(resp.Body)
+
+	return transform(resp.Body)
+}
+
+func transformZipStream() func(in io.ReadCloser) (io.ReadCloser, error) {
+	return func(in io.ReadCloser) (io.ReadCloser, error) {
+		defer in.Close()
+		buf := new(bytes.Buffer)
+		_, err := io.Copy(buf, in)
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(buf), nil
+	}
+}

packer/plugin-getter/plugins.go

@@ -31,6 +31,10 @@ import (
 	"golang.org/x/mod/semver"
 )
 
+const JSONExtension = ".json"
+
+var HTTPFailure = errors.New("http call failed to releases.hashicorp.com failed")
+
 type Requirements []*Requirement
 
 // Requirement describes a required plugin and how it is installed. Usually a list
@@ -537,6 +541,10 @@ func (gp *GetOptions) Version() string {
 	return "v" + gp.version.String()
 }
 
+func (gp *GetOptions) VersionString() string {
+	return gp.version.String()
+}
+
 // A Getter helps get the appropriate files to download a binary.
 type Getter interface {
 	// Get allows Packer to know more information about releases of a plugin in
@@ -586,6 +594,8 @@ type Getter interface {
 	//    packer-plugin-amazon_v1.0.0_x5.0_linux_amd64 file that will be checksum
 	//    verified then copied to the correct plugin location.
 	Get(what string, opts GetOptions) (io.ReadCloser, error)
+
+	GetOfficialRelease(what string, opts GetOptions) (io.ReadCloser, error)
 }
 
 type Release struct {
@@ -635,6 +645,32 @@ func (e *ChecksumFileEntry) init(req *Requirement) (err error) {
 	return err
 }
 
+// a file inside will look like so:
+//
+//	packer-plugin-comment_v0.2.12_freebsd_amd64.zip
+func (e *ChecksumFileEntry) initRelease(req *Requirement) (err error) {
+	filename := e.Filename
+	//remove the test line below where hardcoded prefix being used
+	res := strings.TrimPrefix(filename, req.FilenamePrefix())
+	// res now looks like v0.2.12_freebsd_amd64.zip
+
+	e.ext = filepath.Ext(res)
+
+	res = strings.TrimSuffix(res, e.ext)
+	// res now looks like v0.2.12_freebsd_amd64
+
+	parts := strings.Split(res, "_")
+	// ["v0.2.12", "freebsd", "amd64"]
+	if len(parts) < 3 {
+		return fmt.Errorf("malformed filename expected %s{version}_{os}_{arch}", req.FilenamePrefix())
+	}
+
+	e.binVersion, e.os, e.arch = parts[0], parts[1], parts[2]
+	e.binVersion = strings.TrimPrefix(e.binVersion, "v")
+
+	return err
+}
+
 func (e *ChecksumFileEntry) validate(expectedVersion string, installOpts BinaryInstallationOptions) error {
 	if e.binVersion != expectedVersion {
 		return fmt.Errorf("wrong version: '%s' does not match expected %s ", e.binVersion, expectedVersion)
@@ -646,16 +682,44 @@ func (e *ChecksumFileEntry) validate(expectedVersion string, installOpts BinaryI
 	return installOpts.CheckProtocolVersion(e.protVersion)
 }
 
+func (e *ChecksumFileEntry) validateRelease(expectedVersion string, installOpts BinaryInstallationOptions) error {
+	if e.binVersion != expectedVersion {
+		return fmt.Errorf("wrong version: '%s' does not match expected %s ", e.binVersion, expectedVersion)
+	}
+	if e.os != installOpts.OS || e.arch != installOpts.ARCH {
+		return fmt.Errorf("wrong system, expected %s_%s got %s_%s", installOpts.OS, installOpts.ARCH, e.os, e.arch)
+	}
+
+	return nil
+}
+
 func ParseChecksumFileEntries(f io.Reader) ([]ChecksumFileEntry, error) {
 	var entries []ChecksumFileEntry
 	return entries, json.NewDecoder(f).Decode(&entries)
 }
 
 func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) {
+	newInstall, err := pr.InstallOfficial(opts)
+	if err != nil {
+		if errors.Is(err, HTTPFailure) {
+			log.Printf("[TRACE] Failure from release site, trying to install from gitHub.com %s", pr.Identifier)
+			newInstall, err = pr.InstallFromGitHub(opts)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	return newInstall, nil
+}
+
+func (pr *Requirement) InstallFromGitHub(opts InstallOptions) (*Installation, error) {
 
 	getters := opts.Getters
 
-	log.Printf("[TRACE] getting available versions for the %s plugin", pr.Identifier)
+	log.Printf("[TRACE] getting available versions for the %s plugin from github.com", pr.Identifier)
 	versions := goversion.Collection{}
 	var errs *multierror.Error
 	for _, getter := range getters {
@@ -966,9 +1030,347 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error)
 	return nil, errs
 }
 
+func (pr *Requirement) InstallOfficial(opts InstallOptions) (*Installation, error) {
+
+	getters := opts.Getters
+
+	log.Printf("[TRACE] getting available versions for the %s plugin from releases.hasicorp.com", pr.Identifier)
+	versions := goversion.Collection{}
+	var errs *multierror.Error
+	for _, getter := range getters {
+
+		releasesFile, err := getter.GetOfficialRelease("releases", GetOptions{
+			PluginRequirement:         pr,
+			BinaryInstallationOptions: opts.BinaryInstallationOptions,
+		})
+		if err != nil {
+			if errors.Is(err, HTTPFailure) {
+				return nil, err
+			}
+			errs = multierror.Append(errs, err)
+			log.Printf("[TRACE] %s", err.Error())
+			continue
+		}
+
+		releases, err := ParseReleases(releasesFile)
+		if err != nil {
+			err := fmt.Errorf("could not parse release: %w", err)
+			errs = multierror.Append(errs, err)
+			log.Printf("[TRACE] %s", err.Error())
+			continue
+		}
+		if len(releases) == 0 {
+			err := fmt.Errorf("no release found")
+			errs = multierror.Append(errs, err)
+			log.Printf("[TRACE] %s", err.Error())
+			continue
+		}
+		for _, release := range releases {
+			v, err := goversion.NewVersion(release.Version)
+			if err != nil {
+				err := fmt.Errorf("could not parse release version %s. %w", release.Version, err)
+				errs = multierror.Append(errs, err)
+				log.Printf("[TRACE] %s, ignoring it", err.Error())
+				continue
+			}
+			if pr.VersionConstraints.Check(v) {
+				versions = append(versions, v)
+			}
+		}
+		if len(versions) == 0 {
+			err := fmt.Errorf("no matching version found in releases. In %v", releases)
+			errs = multierror.Append(errs, err)
+			log.Printf("[TRACE] %s", err.Error())
+			continue
+		}
+
+		break
+	}
+
+	if len(versions) == 0 {
+		if errs.Len() == 0 {
+			err := fmt.Errorf("no release version found for constraints: %q", pr.VersionConstraints.String())
+			errs = multierror.Append(errs, err)
+		}
+		return nil, errs
+	}
+
+	// Here we want to try every release in order, starting from the highest one
+	// that matches the requirements. The system and protocol version need to
+	// match too.
+	sort.Sort(sort.Reverse(versions))
+	log.Printf("[DEBUG] will try to install: %s", versions)
+
+	for _, version := range versions {
+		//TODO(azr): split in its own InstallVersion(version, opts) function
+
+		outputFolder := filepath.Join(
+			// Pick last folder as it's the one with the highest priority
+			opts.PluginDirectory,
+			// add expected full path
+			filepath.Join(pr.Identifier.Parts()...),
+		)
+
+		log.Printf("[TRACE] fetching checksums file for the %q version of the %s plugin in %q...", version, pr.Identifier, outputFolder)
+
+		var checksum *FileChecksum
+		for _, getter := range getters {
+			if checksum != nil {
+				break
+			}
+			for _, checksummer := range opts.Checksummers {
+				if checksum != nil {
+					break
+				}
+
+				checksumFile, err := getter.GetOfficialRelease(checksummer.Type, GetOptions{
+					PluginRequirement:         pr,
+					BinaryInstallationOptions: opts.BinaryInstallationOptions,
+					version:                   version,
+				})
+				if err != nil {
+					if errors.Is(err, HTTPFailure) {
+						return nil, err
+					}
+					err := fmt.Errorf("could not get %s checksum file for %s version %s. Is the file present on the release and correctly named ? %w", checksummer.Type, pr.Identifier, version, err)
+					errs = multierror.Append(errs, err)
+					log.Printf("[TRACE] %s", err)
+					continue
+				}
+
+				entries, err := ParseChecksumFileEntries(checksumFile)
+				_ = checksumFile.Close()
+				if err != nil {
+					err := fmt.Errorf("could not parse %s checksumfile: %v. Make sure the checksum file contains a checksum and a binary filename per line", checksummer.Type, err)
+					errs = multierror.Append(errs, err)
+					log.Printf("[TRACE] %s", err)
+					continue
+				}
+
+				for _, entry := range entries {
+					if filepath.Ext(entry.Filename) == JSONExtension {
+						continue
+					}
+					if err := entry.initRelease(pr); err != nil {
+						err := fmt.Errorf("could not parse checksum filename %s. Is it correctly formatted ? %s", entry.Filename, err)
+						errs = multierror.Append(errs, err)
+						log.Printf("[TRACE] %s", err)
+						continue
+					}
+
+					if err := entry.validateRelease(version.String(), opts.BinaryInstallationOptions); err != nil {
+						log.Printf("#### Got error %v", err)
+						continue
+					}
+
+					log.Printf("[TRACE] About to get: %s", entry.Filename)
+
+					cs, err := checksummer.ParseChecksum(strings.NewReader(entry.Checksum))
+					if err != nil {
+						err := fmt.Errorf("could not parse %s checksum: %s. Make sure the checksum file contains the checksum and only the checksum", checksummer.Type, err)
+						errs = multierror.Append(errs, err)
+						log.Printf("[TRACE] %s", err)
+						continue
+					}
+
+					checksum = &FileChecksum{
+						Filename:    entry.Filename,
+						Expected:    cs,
+						Checksummer: checksummer,
+					}
+
+					log.Printf("#### getChecksum filename: %s", checksum.Filename)
+					expectedZipFilename := checksum.Filename
+					expectedBinaryFilename := strings.TrimSuffix(expectedZipFilename, filepath.Ext(expectedZipFilename)) + opts.BinaryInstallationOptions.Ext
+					outputFileName := filepath.Join(outputFolder, expectedBinaryFilename)
+
+					log.Printf("#### output filename: %s", outputFileName)
+
+					for _, potentialChecksumer := range opts.Checksummers {
+						// First check if a local checksum file is already here in the expected
+						// download folder. Here we want to download a binary so we only check
+						// for an existing checksum file from the folder we want to download
+						// into.
+						cs, err := potentialChecksumer.GetCacheChecksumOfFile(outputFileName)
+						if err == nil && len(cs) > 0 {
+							localChecksum := &FileChecksum{
+								Expected:    cs,
+								Checksummer: potentialChecksumer,
+							}
+
+							log.Printf("[TRACE] found a pre-existing %q checksum file", potentialChecksumer.Type)
+							// if outputFile is there and matches the checksum: do nothing more.
+							if err := localChecksum.ChecksumFile(localChecksum.Expected, outputFileName); err == nil && !opts.Force {
+								log.Printf("[INFO] %s v%s plugin is already correctly installed in %q", pr.Identifier, version, outputFileName)
+								return nil, nil // success
+							}
+						}
+					}
+
+					for _, getter := range getters {
+						// start fetching binary
+						remoteZipFile, err := getter.GetOfficialRelease("zip", GetOptions{
+							PluginRequirement:         pr,
+							BinaryInstallationOptions: opts.BinaryInstallationOptions,
+							version:                   version,
+							expectedZipFilename:       expectedZipFilename,
+						})
+						if err != nil {
+							if errors.Is(err, HTTPFailure) {
+								return nil, err
+							}
+							errs = multierror.Append(errs,
+								fmt.Errorf("could not get binary for %s version %s. Is the file present on the release and correctly named ? %s",
+									pr.Identifier, version, err))
+							continue
+						}
+						// create temporary file that will receive a temporary binary.zip
+						tmpFile, err := tmp.File("packer-plugin-*.zip")
+						if err != nil {
+							err = fmt.Errorf("could not create temporary file to download plugin: %w", err)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+						defer func() {
+							tmpFilePath := tmpFile.Name()
+							tmpFile.Close()
+							os.Remove(tmpFilePath)
+						}()
+						// write binary to tmp file
+						_, err = io.Copy(tmpFile, remoteZipFile)
+						_ = remoteZipFile.Close()
+						if err != nil {
+							err := fmt.Errorf("Error getting plugin, trying another getter: %w", err)
+							errs = multierror.Append(errs, err)
+							continue
+						}
+						if _, err := tmpFile.Seek(0, 0); err != nil {
+							err := fmt.Errorf("Error seeking beginning of temporary file for checksumming, continuing: %w", err)
+							errs = multierror.Append(errs, err)
+							continue
+						}
+						// verify that the checksum for the zip is what we expect.
+						if err := checksum.Checksummer.Checksum(checksum.Expected, tmpFile); err != nil {
+							err := fmt.Errorf("%w. Is the checksum file correct ? Is the binary file correct ?", err)
+							errs = multierror.Append(errs, err)
+							continue
+						}
+						zr, err := zip.OpenReader(tmpFile.Name())
+						if err != nil {
+							errs = multierror.Append(errs, fmt.Errorf("zip : %v", err))
+							return nil, errs
+						}
+
+						var copyFrom io.ReadCloser
+						for _, f := range zr.File {
+							if f.Name != "terraform-provider-akamai_v8.0.0" {
+								continue
+							}
+							copyFrom, err = f.Open()
+							if err != nil {
+								errs = multierror.Append(errs, fmt.Errorf("failed to open temp file: %w", err))
+								return nil, errs
+							}
+							break
+						}
+						if copyFrom == nil {
+							err := fmt.Errorf("could not find a %q file in zipfile", expectedBinaryFilename)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+
+						var outputFileData bytes.Buffer
+						if _, err := io.Copy(&outputFileData, copyFrom); err != nil {
+							err := fmt.Errorf("extract file: %w", err)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+						tmpBinFileName := filepath.Join(os.TempDir(), expectedBinaryFilename)
+						tmpOutputFile, err := os.OpenFile(tmpBinFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+						if err != nil {
+							err = fmt.Errorf("could not create temporary file to download plugin: %w", err)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+						defer func() {
+							os.Remove(tmpBinFileName)
+						}()
+
+						if _, err := tmpOutputFile.Write(outputFileData.Bytes()); err != nil {
+							err := fmt.Errorf("extract file: %w", err)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+						tmpOutputFile.Close()
+
+						if err := checkVersion(tmpBinFileName, pr.Identifier.String(), version); err != nil {
+							errs = multierror.Append(errs, err)
+							var continuableError *ContinuableInstallError
+							if errors.As(err, &continuableError) {
+								continue
+							}
+							return nil, errs
+						}
+
+						// create directories if need be
+						if err := os.MkdirAll(outputFolder, 0755); err != nil {
+							err := fmt.Errorf("could not create plugin folder %q: %w", outputFolder, err)
+							errs = multierror.Append(errs, err)
+							log.Printf("[TRACE] %s", err.Error())
+							return nil, errs
+						}
+						outputFile, err := os.OpenFile(outputFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+						if err != nil {
+							err = fmt.Errorf("could not create final plugin binary file: %w", err)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+						if _, err := outputFile.Write(outputFileData.Bytes()); err != nil {
+							err = fmt.Errorf("could not write final plugin binary file: %w", err)
+							errs = multierror.Append(errs, err)
+							return nil, errs
+						}
+						outputFile.Close()
+
+						cs, err := checksum.Checksummer.Sum(&outputFileData)
+						if err != nil {
+							err := fmt.Errorf("failed to checksum binary file: %s", err)
+							errs = multierror.Append(errs, err)
+							log.Printf("[WARNING] %v, ignoring", err)
+						}
+						if err := os.WriteFile(outputFileName+checksum.Checksummer.FileExt(), []byte(hex.EncodeToString(cs)), 0644); err != nil {
+							err := fmt.Errorf("failed to write local binary checksum file: %s", err)
+							errs = multierror.Append(errs, err)
+							log.Printf("[WARNING] %v, ignoring", err)
+							os.Remove(outputFileName)
+							continue
+						}
+
+						// Success !!
+						return &Installation{
+							BinaryPath: strings.ReplaceAll(outputFileName, "\\", "/"),
+							Version:    "v" + version.String(),
+						}, nil
+					}
+				}
+			}
+		}
+	}
+
+	if errs.ErrorOrNil() == nil {
+		err := fmt.Errorf("could not find a local nor a remote checksum for plugin %q %q", pr.Identifier, pr.VersionConstraints)
+		errs = multierror.Append(errs, err)
+	}
+	errs = multierror.Append(errs, fmt.Errorf("could not install any compatible version of plugin %q", pr.Identifier))
+	return nil, errs
+
+	return nil, errs
+}
+
 func GetPluginDescription(pluginPath string) (pluginsdk.SetDescription, error) {
 	out, err := exec.Command(pluginPath, "describe").Output()
 	if err != nil {
+		log.Printf("#### could not get plugin description for %q %q", pluginPath, err)
 		return pluginsdk.SetDescription{}, err
 	}