From 3845186c4c074d4206c6817628253912d447f189 Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Thu, 6 Jun 2019 11:13:17 +0530 Subject: [PATCH 1/7] feat: add feature to import user-data from a file Signed-off-by: Pratyush singhal --- builder/googlecompute/config.go | 1 + builder/googlecompute/step_create_instance.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index df55d99d2..41444d757 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -62,6 +62,7 @@ type Config struct { Subnetwork string `mapstructure:"subnetwork"` Tags []string `mapstructure:"tags"` UseInternalIP bool `mapstructure:"use_internal_ip"` + UserdataFile string `mapstructure:"userdata_file"` Zone string `mapstructure:"zone"` Account AccountFile diff --git a/builder/googlecompute/step_create_instance.go b/builder/googlecompute/step_create_instance.go index 5ed854670..5fd0de5ce 100644 --- a/builder/googlecompute/step_create_instance.go +++ b/builder/googlecompute/step_create_instance.go @@ -19,7 +19,6 @@ type StepCreateInstance struct { func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) (map[string]string, error) { instanceMetadata := make(map[string]string) var err error - // Copy metadata from config. for k, v := range c.Metadata { instanceMetadata[k] = v @@ -45,6 +44,13 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) } else if wrappedStartupScript, exists := instanceMetadata[StartupScriptKey]; exists { instanceMetadata[StartupWrappedScriptKey] = wrappedStartupScript } + + if c.UserdataFile != "" { + var content []byte + content, err = ioutil.ReadFile(c.UserdataFile) + instanceMetadata["user-data"] = string(content) + } + if sourceImage.IsWindows() { // Windows startup script support is not yet implemented. // Mark the startup script as done. From 1e1af353416ccbee6fd26f8ffda4e92a0589bbbe Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Thu, 6 Jun 2019 16:14:57 +0530 Subject: [PATCH 2/7] refactor: replace userdata_files with generic metadata_files map Signed-off-by: Pratyush singhal --- builder/googlecompute/config.go | 2 +- builder/googlecompute/step_create_instance.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index 41444d757..7c78a93cd 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -62,7 +62,7 @@ type Config struct { Subnetwork string `mapstructure:"subnetwork"` Tags []string `mapstructure:"tags"` UseInternalIP bool `mapstructure:"use_internal_ip"` - UserdataFile string `mapstructure:"userdata_file"` + MetadataFiles map[string]string `mapstructure:"metadata_files"` Zone string `mapstructure:"zone"` Account AccountFile diff --git a/builder/googlecompute/step_create_instance.go b/builder/googlecompute/step_create_instance.go index 5fd0de5ce..0688d6280 100644 --- a/builder/googlecompute/step_create_instance.go +++ b/builder/googlecompute/step_create_instance.go @@ -45,10 +45,11 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) instanceMetadata[StartupWrappedScriptKey] = wrappedStartupScript } - if c.UserdataFile != "" { + // Read metadata from files specified with metadata_files + for key, value := range c.MetadataFiles { var content []byte - content, err = ioutil.ReadFile(c.UserdataFile) - instanceMetadata["user-data"] = string(content) + content, err = ioutil.ReadFile(value) + instanceMetadata[key] = string(content) } if sourceImage.IsWindows() { From 529dff0abbc7fc33f42541b7b422caf7a4bdf452 Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Tue, 11 Jun 2019 16:36:16 +0530 Subject: [PATCH 3/7] refactor: add error handling in createInstanceMetadata method Signed-off-by: Pratyush singhal --- builder/googlecompute/step_create_instance.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/builder/googlecompute/step_create_instance.go b/builder/googlecompute/step_create_instance.go index 0688d6280..9cd26863a 100644 --- a/builder/googlecompute/step_create_instance.go +++ b/builder/googlecompute/step_create_instance.go @@ -16,9 +16,11 @@ type StepCreateInstance struct { Debug bool } -func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) (map[string]string, error) { +func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string, state multistep.StateBag) (map[string]string, error) { instanceMetadata := make(map[string]string) var err error + ui := state.Get("ui").(packer.Ui) + // Copy metadata from config. for k, v := range c.Metadata { instanceMetadata[k] = v @@ -40,6 +42,11 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) if c.StartupScriptFile != "" { var content []byte content, err = ioutil.ReadFile(c.StartupScriptFile) + if err != nil { + err = fmt.Errorf("Error reading startup script file: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + } instanceMetadata[StartupWrappedScriptKey] = string(content) } else if wrappedStartupScript, exists := instanceMetadata[StartupScriptKey]; exists { instanceMetadata[StartupWrappedScriptKey] = wrappedStartupScript @@ -49,6 +56,11 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) for key, value := range c.MetadataFiles { var content []byte content, err = ioutil.ReadFile(value) + if err != nil { + err = fmt.Errorf("Error getting %s metadata from %s: %s", key, value, err) + state.Put("error", err) + ui.Error(err.Error()) + } instanceMetadata[key] = string(content) } @@ -105,7 +117,7 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) var errCh <-chan error var metadata map[string]string - metadata, err = c.createInstanceMetadata(sourceImage, string(c.Comm.SSHPublicKey)) + metadata, err = c.createInstanceMetadata(sourceImage, string(c.Comm.SSHPublicKey), state) errCh, err = d.RunInstance(&InstanceConfig{ AcceleratorType: c.AcceleratorType, AcceleratorCount: c.AcceleratorCount, From 99a3e9cf0a1f01da0fbc7b2ebbd532660365d71e Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Tue, 11 Jun 2019 16:47:16 +0530 Subject: [PATCH 4/7] chore: update tests for createInstaceMetadata Signed-off-by: Pratyush singhal --- builder/googlecompute/step_create_instance_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/googlecompute/step_create_instance_test.go b/builder/googlecompute/step_create_instance_test.go index e56159138..8468fd92e 100644 --- a/builder/googlecompute/step_create_instance_test.go +++ b/builder/googlecompute/step_create_instance_test.go @@ -303,7 +303,7 @@ func TestCreateInstanceMetadata(t *testing.T) { key := "abcdefgh12345678" // create our metadata - metadata, err := c.createInstanceMetadata(image, key) + metadata, err := c.createInstanceMetadata(image, key, state) assert.True(t, err == nil, "Metadata creation should have succeeded.") @@ -318,7 +318,7 @@ func TestCreateInstanceMetadata_noPublicKey(t *testing.T) { sshKeys := c.Metadata["sshKeys"] // create our metadata - metadata, err := c.createInstanceMetadata(image, "") + metadata, err := c.createInstanceMetadata(image, "", state) assert.True(t, err == nil, "Metadata creation should have succeeded.") From 4a369b4ef1284f072b36264b0c4f7ae842a958a4 Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Tue, 11 Jun 2019 17:45:30 +0530 Subject: [PATCH 5/7] chore: add test for MetadataFiles option Signed-off-by: Pratyush singhal --- builder/googlecompute/config_test.go | 18 +++++++++++++++++- .../googlecompute/step_create_instance_test.go | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/builder/googlecompute/config_test.go b/builder/googlecompute/config_test.go index 4db82040b..f562d4b8f 100644 --- a/builder/googlecompute/config_test.go +++ b/builder/googlecompute/config_test.go @@ -431,7 +431,8 @@ func testConfig(t *testing.T) (config map[string]interface{}, tempAccountFile st "image_licenses": []string{ "test-license", }, - "zone": "us-east1-a", + "metadata_files": map[string]string{}, + "zone": "us-east1-a", } return config, tempAccountFile @@ -484,6 +485,21 @@ func testAccountFile(t *testing.T) string { return tf.Name() } +const testMetadataFileContent = `testMetadata` + +func testMetadataFile(t *testing.T) string { + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + defer tf.Close() + if _, err := tf.Write([]byte(testMetadataFileContent)); err != nil { + t.Fatalf("err: %s", err) + } + + return tf.Name() +} + // This is just some dummy data that doesn't actually work (it was revoked // a long time ago). const testAccountContent = `{}` diff --git a/builder/googlecompute/step_create_instance_test.go b/builder/googlecompute/step_create_instance_test.go index 8468fd92e..8f765d34f 100644 --- a/builder/googlecompute/step_create_instance_test.go +++ b/builder/googlecompute/step_create_instance_test.go @@ -325,3 +325,20 @@ func TestCreateInstanceMetadata_noPublicKey(t *testing.T) { // ensure the ssh metadata hasn't changed assert.Equal(t, metadata["sshKeys"], sshKeys, "Instance metadata should not have been modified") } + +func TestCreateInstanceMetadata_metadataFile(t *testing.T) { + state := testState(t) + c := state.Get("config").(*Config) + image := StubImage("test-image", "test-project", []string{}, 100) + content := testMetadataFileContent + fileName := testMetadataFile(t) + c.MetadataFiles["user-data"] = fileName + + // create our metadata + metadata, err := c.createInstanceMetadata(image, "", state) + + assert.True(t, err == nil, "Metadata creation should have succeeded.") + + // ensure the user-data key in metadata is updated with file content + assert.Equal(t, metadata["user-data"], content, "user-data field of the instance metadata should have been updated.") +} From 6ce6bd8ad33093df6b06e3032d2906b6d70882ff Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Tue, 11 Jun 2019 20:08:03 +0530 Subject: [PATCH 6/7] refactor: add multiError in createInstanceMetadata method to capture multiple errors Signed-off-by: Pratyush singhal --- builder/googlecompute/step_create_instance.go | 21 ++++++++++--------- .../step_create_instance_test.go | 6 +++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/builder/googlecompute/step_create_instance.go b/builder/googlecompute/step_create_instance.go index 9cd26863a..ec056d596 100644 --- a/builder/googlecompute/step_create_instance.go +++ b/builder/googlecompute/step_create_instance.go @@ -16,10 +16,10 @@ type StepCreateInstance struct { Debug bool } -func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string, state multistep.StateBag) (map[string]string, error) { +func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) (map[string]string, *packer.MultiError) { instanceMetadata := make(map[string]string) var err error - ui := state.Get("ui").(packer.Ui) + var errs *packer.MultiError // Copy metadata from config. for k, v := range c.Metadata { @@ -43,9 +43,7 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string, var content []byte content, err = ioutil.ReadFile(c.StartupScriptFile) if err != nil { - err = fmt.Errorf("Error reading startup script file: %s", err) - state.Put("error", err) - ui.Error(err.Error()) + errs = packer.MultiErrorAppend(errs, err) } instanceMetadata[StartupWrappedScriptKey] = string(content) } else if wrappedStartupScript, exists := instanceMetadata[StartupScriptKey]; exists { @@ -57,9 +55,7 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string, var content []byte content, err = ioutil.ReadFile(value) if err != nil { - err = fmt.Errorf("Error getting %s metadata from %s: %s", key, value, err) - state.Put("error", err) - ui.Error(err.Error()) + errs = packer.MultiErrorAppend(errs, err) } instanceMetadata[key] = string(content) } @@ -74,7 +70,7 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string, instanceMetadata[StartupScriptStatusKey] = StartupScriptStatusNotDone } - return instanceMetadata, err + return instanceMetadata, errs } func getImage(c *Config, d Driver) (*Image, error) { @@ -117,7 +113,12 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) var errCh <-chan error var metadata map[string]string - metadata, err = c.createInstanceMetadata(sourceImage, string(c.Comm.SSHPublicKey), state) + metadata, errs := c.createInstanceMetadata(sourceImage, string(c.Comm.SSHPublicKey)) + if errs != nil && len(errs.Errors) > 0 { + state.Put("error", errs.Error()) + ui.Error(errs.Error()) + } + errCh, err = d.RunInstance(&InstanceConfig{ AcceleratorType: c.AcceleratorType, AcceleratorCount: c.AcceleratorCount, diff --git a/builder/googlecompute/step_create_instance_test.go b/builder/googlecompute/step_create_instance_test.go index 8f765d34f..33f0371fb 100644 --- a/builder/googlecompute/step_create_instance_test.go +++ b/builder/googlecompute/step_create_instance_test.go @@ -303,7 +303,7 @@ func TestCreateInstanceMetadata(t *testing.T) { key := "abcdefgh12345678" // create our metadata - metadata, err := c.createInstanceMetadata(image, key, state) + metadata, err := c.createInstanceMetadata(image, key) assert.True(t, err == nil, "Metadata creation should have succeeded.") @@ -318,7 +318,7 @@ func TestCreateInstanceMetadata_noPublicKey(t *testing.T) { sshKeys := c.Metadata["sshKeys"] // create our metadata - metadata, err := c.createInstanceMetadata(image, "", state) + metadata, err := c.createInstanceMetadata(image, "") assert.True(t, err == nil, "Metadata creation should have succeeded.") @@ -335,7 +335,7 @@ func TestCreateInstanceMetadata_metadataFile(t *testing.T) { c.MetadataFiles["user-data"] = fileName // create our metadata - metadata, err := c.createInstanceMetadata(image, "", state) + metadata, err := c.createInstanceMetadata(image, "") assert.True(t, err == nil, "Metadata creation should have succeeded.") From 92af5847a7de1fddb2f4051cdc3ee7ddfe1ee26d Mon Sep 17 00:00:00 2001 From: Pratyush singhal Date: Tue, 11 Jun 2019 21:01:26 +0530 Subject: [PATCH 7/7] refactor: replace *packer.MultiError from type signature of createInstanceMetadata with generic error interface Signed-off-by: Pratyush singhal --- builder/googlecompute/step_create_instance.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/builder/googlecompute/step_create_instance.go b/builder/googlecompute/step_create_instance.go index ec056d596..6004c2fab 100644 --- a/builder/googlecompute/step_create_instance.go +++ b/builder/googlecompute/step_create_instance.go @@ -16,7 +16,7 @@ type StepCreateInstance struct { Debug bool } -func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) (map[string]string, *packer.MultiError) { +func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) (map[string]string, error) { instanceMetadata := make(map[string]string) var err error var errs *packer.MultiError @@ -43,7 +43,7 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) var content []byte content, err = ioutil.ReadFile(c.StartupScriptFile) if err != nil { - errs = packer.MultiErrorAppend(errs, err) + return nil, err } instanceMetadata[StartupWrappedScriptKey] = string(content) } else if wrappedStartupScript, exists := instanceMetadata[StartupScriptKey]; exists { @@ -70,7 +70,10 @@ func (c *Config) createInstanceMetadata(sourceImage *Image, sshPublicKey string) instanceMetadata[StartupScriptStatusKey] = StartupScriptStatusNotDone } - return instanceMetadata, errs + if errs != nil && len(errs.Errors) > 0 { + return instanceMetadata, errs + } + return instanceMetadata, nil } func getImage(c *Config, d Driver) (*Image, error) { @@ -114,9 +117,10 @@ func (s *StepCreateInstance) Run(ctx context.Context, state multistep.StateBag) var errCh <-chan error var metadata map[string]string metadata, errs := c.createInstanceMetadata(sourceImage, string(c.Comm.SSHPublicKey)) - if errs != nil && len(errs.Errors) > 0 { + if errs != nil { state.Put("error", errs.Error()) ui.Error(errs.Error()) + return multistep.ActionHalt } errCh, err = d.RunInstance(&InstanceConfig{