From c5bcb97d06e2d66b9b433ae1bb18f3c832d67173 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 4 Jan 2018 15:04:07 -0800 Subject: [PATCH 1/3] "borrow" access config code from terraform. This gives us a few benefits: * timeout early if metadata service can't be reached * report which auth provider we're using * give much better errors if something goes wrong --- builder/amazon/common/access_config.go | 99 +++++++++++++++++++++----- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go index b569d0314..e3875d26c 100644 --- a/builder/amazon/common/access_config.go +++ b/builder/amazon/common/access_config.go @@ -1,13 +1,16 @@ package common import ( + "errors" "fmt" "log" "os" "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/hashicorp/go-cleanhttp" @@ -16,15 +19,16 @@ import ( // AccessConfig is for common configuration related to AWS access type AccessConfig struct { - AccessKey string `mapstructure:"access_key"` - CustomEndpointEc2 string `mapstructure:"custom_endpoint_ec2"` - MFACode string `mapstructure:"mfa_code"` - ProfileName string `mapstructure:"profile"` - RawRegion string `mapstructure:"region"` - SecretKey string `mapstructure:"secret_key"` - SkipValidation bool `mapstructure:"skip_region_validation"` - Token string `mapstructure:"token"` - session *session.Session + AccessKey string `mapstructure:"access_key"` + CustomEndpointEc2 string `mapstructure:"custom_endpoint_ec2"` + MFACode string `mapstructure:"mfa_code"` + ProfileName string `mapstructure:"profile"` + RawRegion string `mapstructure:"region"` + SecretKey string `mapstructure:"secret_key"` + SkipValidation bool `mapstructure:"skip_region_validation"` + SkipMetadataApiCheck bool `mapstructure:"skip_metadata_api_check"` + Token string `mapstructure:"token"` + session *session.Session } // Config returns a valid aws.Config object for access to AWS services, or @@ -34,14 +38,78 @@ func (c *AccessConfig) Session() (*session.Session, error) { return c.session, nil } - config := aws.NewConfig().WithMaxRetries(11).WithCredentialsChainVerboseErrors(true) + // build a chain provider, lazy-evaluated by aws-sdk + providers := []credentials.Provider{ + &credentials.StaticProvider{Value: credentials.Value{ + AccessKeyID: c.AccessKey, + SecretAccessKey: c.SecretKey, + SessionToken: c.Token, + }}, + &credentials.EnvProvider{}, + &credentials.SharedCredentialsProvider{ + Filename: "", + Profile: c.ProfileName, + }, + } - if c.ProfileName != "" { - if err := os.Setenv("AWS_PROFILE", c.ProfileName); err != nil { - return nil, fmt.Errorf("Set env error: %s", err) + // Build isolated HTTP client to avoid issues with globally-shared settings + client := cleanhttp.DefaultClient() + + // Keep the default timeout (100ms) low as we don't want to wait in non-EC2 environments + client.Timeout = 100 * time.Millisecond + + const userTimeoutEnvVar = "AWS_METADATA_TIMEOUT" + userTimeout := os.Getenv(userTimeoutEnvVar) + if userTimeout != "" { + newTimeout, err := time.ParseDuration(userTimeout) + if err == nil { + if newTimeout.Nanoseconds() > 0 { + client.Timeout = newTimeout + } else { + log.Printf("[WARN] Non-positive value of %s (%s) is meaningless, ignoring", userTimeoutEnvVar, newTimeout.String()) + } + } else { + log.Printf("[WARN] Error converting %s to time.Duration: %s", userTimeoutEnvVar, err) } } + log.Printf("[INFO] Setting AWS metadata API timeout to %s", client.Timeout.String()) + cfg := &aws.Config{ + HTTPClient: client, + } + if !c.SkipMetadataApiCheck { + // Real AWS should reply to a simple metadata request. + // We check it actually does to ensure something else didn't just + // happen to be listening on the same IP:Port + metadataClient := ec2metadata.New(session.New(cfg)) + if metadataClient.Available() { + providers = append(providers, &ec2rolecreds.EC2RoleProvider{ + Client: metadataClient, + }) + log.Print("[INFO] AWS EC2 instance detected via default metadata" + + " API endpoint, EC2RoleProvider added to the auth chain") + } else { + log.Printf("[INFO] Ignoring AWS metadata API endpoint " + + "as it doesn't return any instance-id") + } + } + + creds := credentials.NewChainCredentials(providers) + cp, err := creds.Get() + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" { + return nil, errors.New("No valid credential sources found for AWS Builder. " + + "Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " + + "for more information on providing credentials for the AWS Builder.") + } + + return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err) + } + log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName) + + config := aws.NewConfig().WithMaxRetries(11).WithCredentialsChainVerboseErrors(true) + config = config.WithCredentials(creds) + if c.RawRegion != "" { config = config.WithRegion(c.RawRegion) } else if region := c.metadataRegion(); region != "" { @@ -52,11 +120,6 @@ func (c *AccessConfig) Session() (*session.Session, error) { config = config.WithEndpoint(c.CustomEndpointEc2) } - if c.AccessKey != "" { - config = config.WithCredentials( - credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)) - } - opts := session.Options{ SharedConfigState: session.SharedConfigEnable, Config: *config, From a9b48309c90ae67598af90b4398bbd16958a0abd Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 8 Feb 2018 14:48:04 -0800 Subject: [PATCH 2/3] update amazon authentication docs Correct docs to reflect new behavior. This is largely borrowed from terraform. --- website/source/docs/builders/amazon.html.md | 105 ++++++++++++++------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/website/source/docs/builders/amazon.html.md b/website/source/docs/builders/amazon.html.md index e77e4bf79..7e7b763c9 100644 --- a/website/source/docs/builders/amazon.html.md +++ b/website/source/docs/builders/amazon.html.md @@ -49,49 +49,90 @@ filesystem and data. -## Specifying Amazon Credentials +## Authentication -When you use any of the amazon builders, you must provide credentials to the API -in the form of an access key id and secret. These look like: +The AWS provider offers a flexible means of providing credentials for +authentication. The following methods are supported, in this order, and +explained below: - access key id: AKIAIOSFODNN7EXAMPLE - secret access key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +- Static credentials +- Environment variables +- Shared credentials file +- EC2 Role -If you use other AWS tools you may already have these configured. If so, packer -will try to use them, *unless* they are specified in your packer template. -Credentials are resolved in the following order: +### Static Credentials -1. Values hard-coded in the packer template are always authoritative. -2. *Variables* in the packer template may be resolved from command-line flags - or from environment variables. Please read about [User - Variables](https://www.packer.io/docs/templates/user-variables.html) - for details. -3. If no credentials are found, packer falls back to automatic lookup. +Static credentials can be provided in the form of an access key id and secret. +These look like: -### Automatic Lookup +```json +{ + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "region": "us-east-1", + "type": "amazon-ebs" +} +``` -Packer depends on the [AWS -SDK](https://aws.amazon.com/documentation/sdk-for-go/) to perform automatic -lookup using *credential chains*. In short, the SDK looks for credentials in -the following order: +### Environment variables -1. Environment variables. -2. Shared credentials file. -3. If your application is running on an Amazon EC2 instance, IAM role for Amazon EC2. +You can provide your credentials via the `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY`, environment variables, representing your AWS Access +Key and AWS Secret Key, respectively. Note that setting your AWS credentials +using either these environment variables will override the use of +`AWS_SHARED_CREDENTIALS_FILE` and `AWS_PROFILE`. The `AWS_DEFAULT_REGION` and +`AWS_SESSION_TOKEN` environment variables are also used, if applicable: -Please refer to the SDK's documentation on [specifying -credentials](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials) -for more information. -## Using an IAM Task or Instance Role +Usage: -If AWS keys are not specified in the template, a -[shared credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) -or through environment variables Packer will use credentials provided by -the task's or instance's IAM role, if it has one. +``` +$ export AWS_ACCESS_KEY_ID="anaccesskey" +$ export AWS_SECRET_ACCESS_KEY="asecretkey" +$ export AWS_DEFAULT_REGION="us-west-2" +$ packer build packer.json +``` -The following policy document provides the minimal set permissions necessary for -Packer to work: +### Shared Credentials file + +You can use an AWS credentials file to specify your credentials. The default +location is $HOME/.aws/credentials on Linux and OS X, or +"%USERPROFILE%.aws\credentials" for Windows users. If we fail to detect +credentials inline, or in the environment, Packer will check this location. You +can optionally specify a different location in the configuration by setting the +environment with the `AWS_SHARED_CREDENTIALS_FILE` variable. + +You may also configure the profile to use by setting the `profile` +configuration option, or setting the `AWS_PROFILE` environment variable: + +```json +{ + "profile": "customprofile", + "region": "us-east-1", + "type": "amazon-ebs" +} +``` + + +### IAM Task or Instance Role + +Finally, Packer will use credentials provided by the task's or instance's IAM +role, if it has one. + +This is a preferred approach over any other when running in EC2 as you can +avoid hard coding credentials. Instead these are leased on-the-fly by Packer, +which reduces the chance of leakage. + +The default deadline for the EC2 metadata API endpoint is 100 milliseconds, +which can be overidden by setting the `AWS_METADATA_TIMEOUT` environment +variable. The variable expects a positive golang Time.Duration string, which is +a sequence of decimal numbers and a unit suffix; valid suffixes are `ns` +(nanoseconds), `us` (microseconds), `ms` (milliseconds), `s` (seconds), `m` +(minutes), and `h` (hours). Examples of valid inputs: `100ms`, `250ms`, `1s`, +`2.5s`, `2.5m`, `1m30s`. + +The following policy document provides the minimal set permissions necessary +for Packer to work: ``` json { From c11627fba7a0e88a921e7510e5203af3a1596b63 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 8 Feb 2018 14:52:51 -0800 Subject: [PATCH 3/3] document new skip_metadata_api_check option --- website/source/docs/builders/amazon-chroot.html.md | 6 ++++++ website/source/docs/builders/amazon-ebs.html.md | 6 ++++++ website/source/docs/builders/amazon-ebssurrogate.html.md | 6 ++++++ website/source/docs/builders/amazon-ebsvolume.html.md | 6 ++++++ website/source/docs/builders/amazon-instance.html.md | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/website/source/docs/builders/amazon-chroot.html.md b/website/source/docs/builders/amazon-chroot.html.md index a983305c2..1757b06bf 100644 --- a/website/source/docs/builders/amazon-chroot.html.md +++ b/website/source/docs/builders/amazon-chroot.html.md @@ -243,6 +243,12 @@ each category, the available configuration keys are alphabetized. of the `source_ami` unless `from_scratch` is `true`, in which case this field must be defined. +- `skip_metadata_api_check` - (boolean) Skip the AWS Metadata API check. + Useful for AWS API implementations that do not have a metadata API + endpoint. Setting to `true` prevents Packer from authenticating via the + Metadata API. You may need to use other authentication methods like static + credentials, configuration variables, or environment variables. + - `skip_region_validation` (boolean) - Set to true if you want to skip validation of the `ami_regions` configuration option. Default `false`. diff --git a/website/source/docs/builders/amazon-ebs.html.md b/website/source/docs/builders/amazon-ebs.html.md index 6b48621b4..c83698569 100644 --- a/website/source/docs/builders/amazon-ebs.html.md +++ b/website/source/docs/builders/amazon-ebs.html.md @@ -244,6 +244,12 @@ builder. in case Packer exits ungracefully. Possible values are "stop" and "terminate", default is `stop`. +- `skip_metadata_api_check` - (boolean) Skip the AWS Metadata API check. + Useful for AWS API implementations that do not have a metadata API + endpoint. Setting to `true` prevents Packer from authenticating via the + Metadata API. You may need to use other authentication methods like static + credentials, configuration variables, or environment variables. + - `skip_region_validation` (boolean) - Set to true if you want to skip validation of the region configuration option. Default `false`. diff --git a/website/source/docs/builders/amazon-ebssurrogate.html.md b/website/source/docs/builders/amazon-ebssurrogate.html.md index 7b995fa87..aeffe2f09 100644 --- a/website/source/docs/builders/amazon-ebssurrogate.html.md +++ b/website/source/docs/builders/amazon-ebssurrogate.html.md @@ -237,6 +237,12 @@ builder. incase packer exits ungracefully. Possible values are "stop" and "terminate", default is `stop`. +- `skip_metadata_api_check` - (boolean) Skip the AWS Metadata API check. + Useful for AWS API implementations that do not have a metadata API + endpoint. Setting to `true` prevents Packer from authenticating via the + Metadata API. You may need to use other authentication methods like static + credentials, configuration variables, or environment variables. + - `skip_region_validation` (boolean) - Set to true if you want to skip validation of the region configuration option. Default `false`. diff --git a/website/source/docs/builders/amazon-ebsvolume.html.md b/website/source/docs/builders/amazon-ebsvolume.html.md index b500aa2bd..174673ed0 100644 --- a/website/source/docs/builders/amazon-ebsvolume.html.md +++ b/website/source/docs/builders/amazon-ebsvolume.html.md @@ -156,6 +156,12 @@ builder. in case Packer exits ungracefully. Possible values are `stop` and `terminate`. Defaults to `stop`. +- `skip_metadata_api_check` - (boolean) Skip the AWS Metadata API check. + Useful for AWS API implementations that do not have a metadata API + endpoint. Setting to `true` prevents Packer from authenticating via the + Metadata API. You may need to use other authentication methods like static + credentials, configuration variables, or environment variables. + - `skip_region_validation` (boolean) - Set to `true` if you want to skip validation of the region configuration option. Defaults to `false`. diff --git a/website/source/docs/builders/amazon-instance.html.md b/website/source/docs/builders/amazon-instance.html.md index f8f72708a..299222837 100644 --- a/website/source/docs/builders/amazon-instance.html.md +++ b/website/source/docs/builders/amazon-instance.html.md @@ -248,6 +248,12 @@ builder. The default is `0.0.0.0/0` (ie, allow any IPv4 source). This is only used when `security_group_id` or `security_group_ids` is not specified. +- `skip_metadata_api_check` - (boolean) Skip the AWS Metadata API check. + Useful for AWS API implementations that do not have a metadata API + endpoint. Setting to `true` prevents Packer from authenticating via the + Metadata API. You may need to use other authentication methods like static + credentials, configuration variables, or environment variables. + - `skip_region_validation` (boolean) - Set to true if you want to skip validation of the region configuration option. Defaults to `false`.