From 7b4f1382170b67deccf93d3af34d99ebcd1b6b66 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Fri, 19 Apr 2019 15:17:37 -0700 Subject: [PATCH 01/11] update docs explaining that you need local artifact for vagrant pp --- website/source/docs/post-processors/vagrant.html.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/source/docs/post-processors/vagrant.html.md b/website/source/docs/post-processors/vagrant.html.md index 62cd0ce0f..f62ff5afd 100644 --- a/website/source/docs/post-processors/vagrant.html.md +++ b/website/source/docs/post-processors/vagrant.html.md @@ -150,3 +150,9 @@ The following Docker input artifacts are supported: The `libvirt` provider supports QEMU artifacts built using any these accelerators: none, kvm, tcg, or hvf. + +### VMWare + +If you are using the Vagrant post-processor with the `vmware-esxi` builder, you +must export the builder artifact locally; the Vagrant post-processor will +not work on remote artifacts. From fe7f5343fd7cd1226c2dcda2910eebfc084d01d1 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Fri, 19 Apr 2019 15:57:54 -0700 Subject: [PATCH 02/11] add vmotion warning to vmware remote builder section, and move the remote builder section into a partial referenced by both the vmx and iso builders --- .../docs/builders/vmware-iso.html.md.erb | 64 +----------------- .../docs/builders/vmware-vmx.html.md.erb | 2 + ...lding_on_remote_vsphere_hypervisor.html.md | 66 +++++++++++++++++++ 3 files changed, 69 insertions(+), 63 deletions(-) create mode 100644 website/source/partials/builders/_building_on_remote_vsphere_hypervisor.html.md diff --git a/website/source/docs/builders/vmware-iso.html.md.erb b/website/source/docs/builders/vmware-iso.html.md.erb index 3757cedf0..4c67d26be 100644 --- a/website/source/docs/builders/vmware-iso.html.md.erb +++ b/website/source/docs/builders/vmware-iso.html.md.erb @@ -496,69 +496,7 @@ variables isn't required, however. - `Version` - The Hardware version VMWare will execute this vm under. Also known as the `virtualhw.version`. -## Building on a Remote vSphere Hypervisor - -In addition to using the desktop products of VMware locally to build virtual -machines, Packer can use a remote VMware Hypervisor to build the virtual -machine. - --> **Note:** Packer supports ESXi 5.1 and above. - -Before using a remote vSphere Hypervisor, you need to enable GuestIPHack by -running the following command: - -``` text -esxcli system settings advanced set -o /Net/GuestIPHack -i 1 -``` - -When using a remote VMware Hypervisor, the builder still downloads the ISO and -various files locally, and uploads these to the remote machine. Packer currently -uses SSH to communicate to the ESXi machine rather than the vSphere API. At some -point, the vSphere API may be used. - -Packer also requires VNC to issue boot commands during a build, which may be -disabled on some remote VMware Hypervisors. Please consult the appropriate -documentation on how to update VMware Hypervisor's firewall to allow these -connections. VNC can be disabled by not setting a `boot_command` and setting -`disable_vnc` to `true`. - -To use a remote VMware vSphere Hypervisor to build your virtual machine, fill in -the required `remote_*` configurations: - -- `remote_type` - This must be set to "esx5". - -- `remote_host` - The host of the remote machine. - -Additionally, there are some optional configurations that you'll likely have to -modify as well: - -- `remote_port` - The SSH port of the remote machine - -- `remote_datastore` - The path to the datastore where the VM will be stored - on the ESXi machine. - -- `remote_cache_datastore` - The path to the datastore where supporting files - will be stored during the build on the remote machine. - -- `remote_cache_directory` - The path where the ISO and/or floppy files will - be stored during the build on the remote machine. The path is relative to - the `remote_cache_datastore` on the remote machine. - -- `remote_username` - The SSH username used to access the remote machine. - -- `remote_password` - The SSH password for access to the remote machine. - -- `remote_private_key_file` - The SSH key for access to the remote machine. - -- `format` (string) - Either "ovf", "ova" or "vmx", this specifies the output - format of the exported virtual machine. This defaults to "ovf". - Before using this option, you need to install `ovftool`. This option - currently only works when option remote_type is set to "esx5". - Since ovftool is only capable of password based authentication - `remote_password` must be set when exporting the VM. - -- `vnc_disable_password` - This must be set to "true" when using VNC with - ESXi 6.5 or 6.7. +<%= partial "partials/builders/building_on_remote_vsphere_hypervisor" %> ### VNC port discovery diff --git a/website/source/docs/builders/vmware-vmx.html.md.erb b/website/source/docs/builders/vmware-vmx.html.md.erb index 765ee5f17..362a0bcc8 100644 --- a/website/source/docs/builders/vmware-vmx.html.md.erb +++ b/website/source/docs/builders/vmware-vmx.html.md.erb @@ -360,3 +360,5 @@ Ubuntu 12.04 installer: For more examples of various boot commands, see the sample projects from our [community templates page](/community-tools.html#templates). + +<%= partial "partials/builders/building_on_remote_vsphere_hypervisor" %> diff --git a/website/source/partials/builders/_building_on_remote_vsphere_hypervisor.html.md b/website/source/partials/builders/_building_on_remote_vsphere_hypervisor.html.md new file mode 100644 index 000000000..8c04d4abb --- /dev/null +++ b/website/source/partials/builders/_building_on_remote_vsphere_hypervisor.html.md @@ -0,0 +1,66 @@ +## Building on a Remote vSphere Hypervisor + +In addition to using the desktop products of VMware locally to build virtual +machines, Packer can use a remote VMware Hypervisor to build the virtual +machine. + +-> **Note:** Packer supports ESXi 5.1 and above. + +Before using a remote vSphere Hypervisor, you need to enable GuestIPHack by +running the following command: + +``` text +esxcli system settings advanced set -o /Net/GuestIPHack -i 1 +``` + +When using a remote VMware Hypervisor, the builder still downloads the ISO and +various files locally, and uploads these to the remote machine. Packer currently +uses SSH to communicate to the ESXi machine rather than the vSphere API. At some +point, the vSphere API may be used. + +Packer also requires VNC to issue boot commands during a build, which may be +disabled on some remote VMware Hypervisors. Please consult the appropriate +documentation on how to update VMware Hypervisor's firewall to allow these +connections. VNC can be disabled by not setting a `boot_command` and setting +`disable_vnc` to `true`. + +Please note that you should disable vMotion for the host you intend to run +Packer builds on; a vMotion event will cause the Packer build to fail. + +To use a remote VMware vSphere Hypervisor to build your virtual machine, fill in +the required `remote_*` configurations: + +- `remote_type` - This must be set to "esx5". + +- `remote_host` - The host of the remote machine. + +Additionally, there are some optional configurations that you'll likely have to +modify as well: + +- `remote_port` - The SSH port of the remote machine + +- `remote_datastore` - The path to the datastore where the VM will be stored + on the ESXi machine. + +- `remote_cache_datastore` - The path to the datastore where supporting files + will be stored during the build on the remote machine. + +- `remote_cache_directory` - The path where the ISO and/or floppy files will + be stored during the build on the remote machine. The path is relative to + the `remote_cache_datastore` on the remote machine. + +- `remote_username` - The SSH username used to access the remote machine. + +- `remote_password` - The SSH password for access to the remote machine. + +- `remote_private_key_file` - The SSH key for access to the remote machine. + +- `format` (string) - Either "ovf", "ova" or "vmx", this specifies the output + format of the exported virtual machine. This defaults to "ovf". + Before using this option, you need to install `ovftool`. This option + currently only works when option remote_type is set to "esx5". + Since ovftool is only capable of password based authentication + `remote_password` must be set when exporting the VM. + +- `vnc_disable_password` - This must be set to "true" when using VNC with + ESXi 6.5 or 6.7. \ No newline at end of file From 8c9f2f5e1c60c2476dcb87e4650409d89af504e7 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Fri, 19 Apr 2019 16:35:15 -0700 Subject: [PATCH 03/11] add a table explaining what to expect with different ssh options for the amazon builders. Also convert builder docs to .erb to support partials --- ...mazon-chroot.html.md => amazon-chroot.html.md.erb} | 0 .../{amazon-ebs.html.md => amazon-ebs.html.md.erb} | 8 +++++--- ...rogate.html.md => amazon-ebssurrogate.html.md.erb} | 7 +++++-- ...ebsvolume.html.md => amazon-ebsvolume.html.md.erb} | 3 +++ ...n-instance.html.md => amazon-instance.html.md.erb} | 3 +++ .../builders/_aws-ssh-differentiation-table.html.md | 11 +++++++++++ 6 files changed, 27 insertions(+), 5 deletions(-) rename website/source/docs/builders/{amazon-chroot.html.md => amazon-chroot.html.md.erb} (100%) rename website/source/docs/builders/{amazon-ebs.html.md => amazon-ebs.html.md.erb} (99%) rename website/source/docs/builders/{amazon-ebssurrogate.html.md => amazon-ebssurrogate.html.md.erb} (99%) rename website/source/docs/builders/{amazon-ebsvolume.html.md => amazon-ebsvolume.html.md.erb} (99%) rename website/source/docs/builders/{amazon-instance.html.md => amazon-instance.html.md.erb} (99%) create mode 100644 website/source/partials/builders/_aws-ssh-differentiation-table.html.md diff --git a/website/source/docs/builders/amazon-chroot.html.md b/website/source/docs/builders/amazon-chroot.html.md.erb similarity index 100% rename from website/source/docs/builders/amazon-chroot.html.md rename to website/source/docs/builders/amazon-chroot.html.md.erb diff --git a/website/source/docs/builders/amazon-ebs.html.md b/website/source/docs/builders/amazon-ebs.html.md.erb similarity index 99% rename from website/source/docs/builders/amazon-ebs.html.md rename to website/source/docs/builders/amazon-ebs.html.md.erb index 7736b797f..2c1494f06 100644 --- a/website/source/docs/builders/amazon-ebs.html.md +++ b/website/source/docs/builders/amazon-ebs.html.md.erb @@ -226,11 +226,11 @@ builder. Unlimited - even for instances that would usually qualify for the [AWS Free Tier](https://aws.amazon.com/free/). -- `encrypt_boot` (boolean) - Whether or not to encrypt the resulting AMI when - copying a provisioned instance to an AMI. By default, Packer will keep the +- `encrypt_boot` (boolean) - Whether or not to encrypt the resulting AMI when + copying a provisioned instance to an AMI. By default, Packer will keep the encryption setting to what it was in the source image. Setting `false` will result in an unencrypted image, and `true` will result in an encrypted one. - + - `force_delete_snapshot` (boolean) - Force Packer to delete snapshots associated with AMIs, which have been deregistered by `force_deregister`. Default `false`. @@ -726,3 +726,5 @@ be easily added to the provisioner section. } ``` + +<%= partial "partials/builders/aws-ssh-differentiation-table" %> diff --git a/website/source/docs/builders/amazon-ebssurrogate.html.md b/website/source/docs/builders/amazon-ebssurrogate.html.md.erb similarity index 99% rename from website/source/docs/builders/amazon-ebssurrogate.html.md rename to website/source/docs/builders/amazon-ebssurrogate.html.md.erb index 093a29523..40d28645a 100644 --- a/website/source/docs/builders/amazon-ebssurrogate.html.md +++ b/website/source/docs/builders/amazon-ebssurrogate.html.md.erb @@ -215,8 +215,8 @@ builder. Unlimited - even for instances that would usually qualify for the [AWS Free Tier](https://aws.amazon.com/free/). -- `encrypt_boot` (boolean) - Whether or not to encrypt the resulting AMI when - copying a provisioned instance to an AMI. By default, Packer will keep the +- `encrypt_boot` (boolean) - Whether or not to encrypt the resulting AMI when + copying a provisioned instance to an AMI. By default, Packer will keep the encryption setting to what it was in the source image. Setting `false` will result in an unencrypted image, and `true` will result in an encrypted one. @@ -626,3 +626,6 @@ These source AMIs may include volumes that are not flagged to be destroyed on termination of the instance building the new image. In addition to those volumes created by this builder, any volumes inn the source AMI which are not marked for deletion on termination will remain in your account. + + +<%= partial "partials/builders/aws-ssh-differentiation-table" %> \ No newline at end of file diff --git a/website/source/docs/builders/amazon-ebsvolume.html.md b/website/source/docs/builders/amazon-ebsvolume.html.md.erb similarity index 99% rename from website/source/docs/builders/amazon-ebsvolume.html.md rename to website/source/docs/builders/amazon-ebsvolume.html.md.erb index 82138690c..2c643c9ba 100644 --- a/website/source/docs/builders/amazon-ebsvolume.html.md +++ b/website/source/docs/builders/amazon-ebsvolume.html.md.erb @@ -564,3 +564,6 @@ These source AMIs may include volumes that are not flagged to be destroyed on termination of the instance building the new image. In addition to those volumes created by this builder, any volumes inn the source AMI which are not marked for deletion on termination will remain in your account. + + +<%= partial "partials/builders/aws-ssh-differentiation-table" %> \ No newline at end of file diff --git a/website/source/docs/builders/amazon-instance.html.md b/website/source/docs/builders/amazon-instance.html.md.erb similarity index 99% rename from website/source/docs/builders/amazon-instance.html.md rename to website/source/docs/builders/amazon-instance.html.md.erb index 8ace80559..c03390b99 100644 --- a/website/source/docs/builders/amazon-instance.html.md +++ b/website/source/docs/builders/amazon-instance.html.md.erb @@ -705,3 +705,6 @@ this: ``` You may wish to constrain the resource to a specific bucket. + + +<%= partial "partials/builders/aws-ssh-differentiation-table" %> \ No newline at end of file diff --git a/website/source/partials/builders/_aws-ssh-differentiation-table.html.md b/website/source/partials/builders/_aws-ssh-differentiation-table.html.md new file mode 100644 index 000000000..b1ec6fd07 --- /dev/null +++ b/website/source/partials/builders/_aws-ssh-differentiation-table.html.md @@ -0,0 +1,11 @@ +## Which SSH Options to use: + +This chart breaks down Packer does if you set any of the below SSH options: + +| ssh_password | ssh_private_key_file | ssh_keypair_name | temporary_key_pair_name | Packer will... | +| --- | --- | --- | --- | --- | +| X | - | - | - | ssh authenticating with username and given password | +| - | X | - | - | ssh authenticating with private key file | +| - | X | X | - | ssh authenticating with given private key file and "attaching" the keypair to the instance | +| - | - | - | X | Create a temporary ssh keypair with a particular name, clean it up | +| - | - | - | - | Create a temporary ssh keypair with a default name, clean it up | \ No newline at end of file From 11037968a8f71ed04122b9a49c3d4831a5683a4c Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Fri, 19 Apr 2019 17:03:20 -0700 Subject: [PATCH 04/11] warn users to lock down winrm after provisioning --- website/source/docs/builders/googlecompute.html.md | 3 +++ website/source/docs/builders/hyperv-iso.html.md.erb | 3 +++ website/source/docs/builders/hyperv-vmcx.html.md.erb | 3 +++ website/source/docs/builders/ncloud.html.md | 3 +++ website/source/docs/provisioners/ansible.html.md.erb | 3 +++ website/source/intro/getting-started/build-image.html.md | 3 +++ 6 files changed, 18 insertions(+) diff --git a/website/source/docs/builders/googlecompute.html.md b/website/source/docs/builders/googlecompute.html.md index b8d1233dc..dfbdd514e 100644 --- a/website/source/docs/builders/googlecompute.html.md +++ b/website/source/docs/builders/googlecompute.html.md @@ -166,6 +166,9 @@ setting a valid `account_file` and `project_id`: } ``` +-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: +https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ + This build can take up to 15 min. ### Nested Hypervisor Example diff --git a/website/source/docs/builders/hyperv-iso.html.md.erb b/website/source/docs/builders/hyperv-iso.html.md.erb index c739f077c..ae2a375e0 100644 --- a/website/source/docs/builders/hyperv-iso.html.md.erb +++ b/website/source/docs/builders/hyperv-iso.html.md.erb @@ -876,6 +876,9 @@ Finish proxy after sysprep --> ``` +-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: +https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ + ## Example For Ubuntu Vivid Generation 2 If you are running Windows under virtualization, you may need to create a diff --git a/website/source/docs/builders/hyperv-vmcx.html.md.erb b/website/source/docs/builders/hyperv-vmcx.html.md.erb index a492fa935..34121076c 100644 --- a/website/source/docs/builders/hyperv-vmcx.html.md.erb +++ b/website/source/docs/builders/hyperv-vmcx.html.md.erb @@ -893,6 +893,9 @@ Finish proxy after sysprep --> ``` +-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: +https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ + ## Example For Ubuntu Vivid Generation 2 If you are running Windows under virtualization, you may need to create a diff --git a/website/source/docs/builders/ncloud.html.md b/website/source/docs/builders/ncloud.html.md index ddc91b257..32199cf6e 100644 --- a/website/source/docs/builders/ncloud.html.md +++ b/website/source/docs/builders/ncloud.html.md @@ -90,6 +90,9 @@ Here is a basic example for windows server. ] } +-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: +https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ + Here is a basic example for linux server. { diff --git a/website/source/docs/provisioners/ansible.html.md.erb b/website/source/docs/provisioners/ansible.html.md.erb index 3d8edf181..48bbc1afd 100644 --- a/website/source/docs/provisioners/ansible.html.md.erb +++ b/website/source/docs/provisioners/ansible.html.md.erb @@ -322,6 +322,9 @@ Platform: } ``` +-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: +https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ + ### Post i/o timeout errors If you see diff --git a/website/source/intro/getting-started/build-image.html.md b/website/source/intro/getting-started/build-image.html.md index bb9f7a6a3..672fe8dac 100644 --- a/website/source/intro/getting-started/build-image.html.md +++ b/website/source/intro/getting-started/build-image.html.md @@ -404,6 +404,9 @@ Start-Service -Name WinRM ``` +-> **Warning:** Please note that if you're setting up WinRM for provisioning, you'll probably want to turn it off or restrict its permissions as part of a shutdown script at the end of Packer's provisioning process. For more details on the why/how, check out this useful blog post and the associated code: +https://cloudywindows.io/post/winrm-for-provisioning---close-the-door-on-the-way-out-eh/ + Save the above code in a file named `bootstrap_win.txt`. -> **A quick aside/warning:**
From 65545073f8b80136d32cc6fe84fabcb861eb4d27 Mon Sep 17 00:00:00 2001 From: Calle Pettersson Date: Sun, 21 Apr 2019 08:33:34 +0200 Subject: [PATCH 05/11] Fix port number type --- builder/proxmox/step_type_boot_command.go | 4 ++-- builder/proxmox/step_type_boot_command_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/proxmox/step_type_boot_command.go b/builder/proxmox/step_type_boot_command.go index 89d50fb0d..04361ee21 100644 --- a/builder/proxmox/step_type_boot_command.go +++ b/builder/proxmox/step_type_boot_command.go @@ -26,7 +26,7 @@ type stepTypeBootCommand struct { type bootCommandTemplateData struct { HTTPIP string - HTTPPort uint + HTTPPort int } type commandTyper interface { @@ -66,7 +66,7 @@ func (s *stepTypeBootCommand) Run(ctx context.Context, state multistep.StateBag) common.SetHTTPIP(httpIP) s.Ctx.Data = &bootCommandTemplateData{ HTTPIP: httpIP, - HTTPPort: state.Get("http_port").(uint), + HTTPPort: state.Get("http_port").(int), } ui.Say("Typing the boot command") diff --git a/builder/proxmox/step_type_boot_command_test.go b/builder/proxmox/step_type_boot_command_test.go index c51b57d73..9514ee0c4 100644 --- a/builder/proxmox/step_type_boot_command_test.go +++ b/builder/proxmox/step_type_boot_command_test.go @@ -106,7 +106,7 @@ func TestTypeBootCommand(t *testing.T) { state := new(multistep.BasicStateBag) state.Put("ui", packer.TestUi(t)) state.Put("config", c.builderConfig) - state.Put("http_port", uint(0)) + state.Put("http_port", int(0)) state.Put("vmRef", proxmox.NewVmRef(1)) state.Put("proxmoxClient", typer) From b56b3dda5faa5b0629829bbca0ccb5089dee590f Mon Sep 17 00:00:00 2001 From: Calle Pettersson Date: Sun, 21 Apr 2019 08:33:51 +0200 Subject: [PATCH 06/11] Fix os parameter documentation --- website/source/docs/builders/proxmox.html.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/source/docs/builders/proxmox.html.md b/website/source/docs/builders/proxmox.html.md index ea8beb58d..2345c1911 100644 --- a/website/source/docs/builders/proxmox.html.md +++ b/website/source/docs/builders/proxmox.html.md @@ -71,8 +71,9 @@ builder. - `sockets` (int) - How many CPU sockets to give the virtual machine. Defaults to `1` -- `os` (string) - The operating system. Can be `linux`, `windows`, `solaris` - or `other`. Defaults to `other`. +- `os` (string) - The operating system. Can be `wxp`, `w2k`, `w2k3`, `w2k8`, + `wvista`, `win7`, `win8`, `win10`, `l24` (Linux 2.4), `l26` (Linux 2.6+), + `solaris` or `other`. Defaults to `other`. - `network_adapters` (array of objects) - Network adapters attached to the virtual machine. Example: From 9eadac3410e80f4a8520c9caf8f52f1f0c3f728e Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Mon, 22 Apr 2019 15:35:00 -0700 Subject: [PATCH 07/11] add guide for uisng jq to strip comments from packer template --- .../guides/use-packer-with-comment.html.md | 77 +++++++++++++++++++ website/source/layouts/guides.erb | 3 + 2 files changed, 80 insertions(+) create mode 100644 website/source/guides/use-packer-with-comment.html.md diff --git a/website/source/guides/use-packer-with-comment.html.md b/website/source/guides/use-packer-with-comment.html.md new file mode 100644 index 000000000..52fbaa7dc --- /dev/null +++ b/website/source/guides/use-packer-with-comment.html.md @@ -0,0 +1,77 @@ +--- +layout: guides +sidebar_current: use-packer-with-comment +page_title: Use jq and Packer to comment your templates - Guides +description: |- + You can add detailed comments beyond the root-level underscore-prefixed field + supported by Packer, and remove them using jq. +--- + +#How to use jq to strip unsupported comments from a Packer template + +One of the biggest complaints we get about packer is that json doesn't use comments. We're in the process of moving to HCL2, the same config language used by Terraform, which does allow comments. But in the meantime, you can add detailed comments beyond the root-level underscore-prefixed field supported by Packer, and remove them using jq. + +Let's say we have a file named commented_template.json + +``` json +{ + "_comment": [ + "this is", + "a multi-line", + "comment" + ], + "builders": [ + { + "_comment": "this is a comment inside a builder", + "type": "null", + "communicator": "none" + } + ], + "_comment": "this is a root level comment", + "provisioners": [ + { + "_comment": "this is a different comment", + "type": "shell", + "_comment": "this is yet another comment", + "inline": ["echo hellooooo"] + } + ] +} +``` + +```sh +jq 'walk(if type == "object" then del(._comment) else . end)' commented_template.json > uncommented_template.json +``` + +will produce a new file containing: + +```json +{ + "builders": [ + { + "type": "null", + "communicator": "none" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": [ + "echo hellooooo" + ] + } + ] +} +``` + +Once you've got your uncommented file, you can call `packer build` on it like +you normally would. + +## The walk function +If your install of jq does not have the walk function and you get an error like + +``` +jq: error: walk/1 is not defined at , +``` + +You can create a file `~/.jq` and add the [walk function](https://github.com/stedolan/jq/blob/ad9fc9f559e78a764aac20f669f23cdd020cd943/src/builtin.jq#L255-L262) to it by hand. diff --git a/website/source/layouts/guides.erb b/website/source/layouts/guides.erb index 1d5653c04..f6e2de6d0 100644 --- a/website/source/layouts/guides.erb +++ b/website/source/layouts/guides.erb @@ -4,6 +4,9 @@ > Veewee to Packer + > + Use jq to strip comments from a Packer template + > Build Immutable Infrastructure with Packer in CI/CD
    From f985f2de62191f1515acde0e58787045935566b5 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Mon, 22 Apr 2019 16:10:35 -0700 Subject: [PATCH 08/11] add docs for how to use the format strings for the isotime function --- .../guides/isotime-template-function.html.md | 89 +++++++++++++++++++ website/source/layouts/guides.erb | 3 + 2 files changed, 92 insertions(+) create mode 100644 website/source/guides/isotime-template-function.html.md diff --git a/website/source/guides/isotime-template-function.html.md b/website/source/guides/isotime-template-function.html.md new file mode 100644 index 000000000..48c084704 --- /dev/null +++ b/website/source/guides/isotime-template-function.html.md @@ -0,0 +1,89 @@ +--- +layout: guides +sidebar_current: isotime-template-function +page_title: Using the isotime template function - Guides +description: |- + It can be a bit confusing to figure out how to format your isotime using the + golang reference date string. Here is a small guide and some examples. +--- + +# Using the Isotime template function with a format string + +The way you format isotime in golang is a bit nontraditional compared to how +you may be used to formatting datetime strings. + +Full docs and examples for the golang time formatting function can be found +[here](https://golang.org/pkg/time/#example_Time_Format) + +However, the formatting basics are worth describing here. From the [golang docs](https://golang.org/pkg/time/#pkg-constants): + + +>These are predefined layouts for use in Time.Format and time.Parse. The +>reference time used in the layouts is the specific time: +> +>Mon Jan 2 15:04:05 MST 2006 +> +>which is Unix time 1136239445. Since MST is GMT-0700, the reference time +>can be thought of as +> +>01/02 03:04:05PM '06 -0700 +> +> To define your own format, write down what the reference time would look like +> formatted your way; see the values of constants like ANSIC, StampMicro or +> Kitchen for examples. The model is to demonstrate what the reference time +> looks like so that the Format and Parse methods can apply the same +> transformation to a general time value. + + +So what does that look like in a Packer template function? + +``` json +{ + "variables": + { + "myvar": "packer-{{isotime \"2006-01-02 03:04:05\"}}" + }, + "builders": [ + { + "type": "null", + "communicator": "none" + } + ], + "provisioners": [ + { + "type": "shell-local", + "inline": ["echo {{ user `myvar`}}"] + } + ] +} +``` + +You can switch out the variables section above with the following examples to +get different timestamps: + +Date only, not time: + +```json + "variables": + { + "myvar": "packer-{{isotime \"2006-01-02\"}}" + }, +``` + +A timestamp down to the millisecond: + +```json + "variables": + { + "myvar": "packer-{{isotime \"Jan-_2-15:04:05.000\"}}" + }, +``` + +Or just the time as it would appear on a digital clock: + +```json + "variables": + { + "myvar": "packer-{{isotime \"3:04PM\"}}" + }, +``` \ No newline at end of file diff --git a/website/source/layouts/guides.erb b/website/source/layouts/guides.erb index 1d5653c04..d285de620 100644 --- a/website/source/layouts/guides.erb +++ b/website/source/layouts/guides.erb @@ -1,6 +1,9 @@ <% wrap_layout :inner do %> <% content_for :sidebar do %>
      + > + Isotime Template Function + > Veewee to Packer From ee743f0b7c7e596adb61ec6e38f068939c423326 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 23 Apr 2019 15:42:59 +0200 Subject: [PATCH 09/11] Update website/source/partials/builders/_aws-ssh-differentiation-table.html.md --- .../partials/builders/_aws-ssh-differentiation-table.html.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/partials/builders/_aws-ssh-differentiation-table.html.md b/website/source/partials/builders/_aws-ssh-differentiation-table.html.md index b1ec6fd07..e56f7d47f 100644 --- a/website/source/partials/builders/_aws-ssh-differentiation-table.html.md +++ b/website/source/partials/builders/_aws-ssh-differentiation-table.html.md @@ -1,6 +1,6 @@ ## Which SSH Options to use: -This chart breaks down Packer does if you set any of the below SSH options: +This chart breaks down what Packer does if you set any of the below SSH options: | ssh_password | ssh_private_key_file | ssh_keypair_name | temporary_key_pair_name | Packer will... | | --- | --- | --- | --- | --- | @@ -8,4 +8,4 @@ This chart breaks down Packer does if you set any of the below SSH options: | - | X | - | - | ssh authenticating with private key file | | - | X | X | - | ssh authenticating with given private key file and "attaching" the keypair to the instance | | - | - | - | X | Create a temporary ssh keypair with a particular name, clean it up | -| - | - | - | - | Create a temporary ssh keypair with a default name, clean it up | \ No newline at end of file +| - | - | - | - | Create a temporary ssh keypair with a default name, clean it up | From e69d95eb37ac3d46640cf423011149fbe0dbf83c Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Tue, 23 Apr 2019 13:48:30 -0700 Subject: [PATCH 10/11] when retries were exhausted in the retry Run, we were returning nil instead of an error. --- common/retry/retry.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/retry/retry.go b/common/retry/retry.go index 2e0cdc0b3..eb8fc4f70 100644 --- a/common/retry/retry.go +++ b/common/retry/retry.go @@ -25,6 +25,8 @@ type Config struct { ShouldRetry func(error) bool } +var RetryExhaustedError error = fmt.Errorf("Function never succeeded in Retry") + // Run fn until context is cancelled up until StartTimeout time has passed. func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error { retryDelay := func() time.Duration { return 2 * time.Second } @@ -43,7 +45,7 @@ func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error for try := 0; ; try++ { var err error if cfg.Tries != 0 && try == cfg.Tries { - return err + return RetryExhaustedError } if err = fn(ctx); err == nil { return nil From 9f1136db77984ab8419d6c516a6261666cef3122 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Wed, 24 Apr 2019 14:03:36 +0200 Subject: [PATCH 11/11] retry: encapsulate & return the last seen error in a RetryExhaustedError --- common/retry/retry.go | 15 +++++++++--- common/retry/retry_test.go | 32 +++++++++++++++++++++---- common/retry/utils_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 common/retry/utils_test.go diff --git a/common/retry/retry.go b/common/retry/retry.go index eb8fc4f70..2b1a4f231 100644 --- a/common/retry/retry.go +++ b/common/retry/retry.go @@ -25,7 +25,16 @@ type Config struct { ShouldRetry func(error) bool } -var RetryExhaustedError error = fmt.Errorf("Function never succeeded in Retry") +type RetryExhaustedError struct { + Err error +} + +func (err *RetryExhaustedError) Error() string { + if err == nil || err.Err == nil { + return "" + } + return fmt.Sprintf("retry count exhausted. Last err: %s", err.Err) +} // Run fn until context is cancelled up until StartTimeout time has passed. func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error { @@ -42,10 +51,10 @@ func (cfg Config) Run(ctx context.Context, fn func(context.Context) error) error startTimeout = time.After(cfg.StartTimeout) } + var err error for try := 0; ; try++ { - var err error if cfg.Tries != 0 && try == cfg.Tries { - return RetryExhaustedError + return &RetryExhaustedError{err} } if err = fn(ctx); err == nil { return nil diff --git a/common/retry/retry_test.go b/common/retry/retry_test.go index 334707c0b..41ecc16c3 100644 --- a/common/retry/retry_test.go +++ b/common/retry/retry_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" "time" + + "github.com/google/go-cmp/cmp" ) func success(context.Context) error { return nil } @@ -18,12 +20,23 @@ var failErr = errors.New("woops !") func fail(context.Context) error { return failErr } +type failOnce bool + +func (ran *failOnce) Run(context.Context) error { + if !*ran { + *ran = true + return failErr + } + return nil +} + func TestConfig_Run(t *testing.T) { cancelledCtx, cancel := context.WithCancel(context.Background()) cancel() type fields struct { StartTimeout time.Duration RetryDelay func() time.Duration + Tries int } type args struct { ctx context.Context @@ -36,26 +49,37 @@ func TestConfig_Run(t *testing.T) { wantErr error }{ {"success", - fields{StartTimeout: time.Second, RetryDelay: nil}, + fields{StartTimeout: time.Second}, args{context.Background(), success}, nil}, {"context cancelled", - fields{StartTimeout: time.Second, RetryDelay: nil}, + fields{StartTimeout: time.Second}, args{cancelledCtx, wait}, context.Canceled}, {"timeout", fields{StartTimeout: 20 * time.Millisecond, RetryDelay: func() time.Duration { return 10 * time.Millisecond }}, args{cancelledCtx, fail}, failErr}, + {"success after one failure", + fields{Tries: 2, RetryDelay: func() time.Duration { return 0 }}, + args{context.Background(), new(failOnce).Run}, + nil}, + {"fail after one failure", + fields{Tries: 1, RetryDelay: func() time.Duration { return 0 }}, + args{context.Background(), new(failOnce).Run}, + &RetryExhaustedError{failErr}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := Config{ StartTimeout: tt.fields.StartTimeout, RetryDelay: tt.fields.RetryDelay, + Tries: tt.fields.Tries, } - if err := cfg.Run(tt.args.ctx, tt.args.fn); err != tt.wantErr { - t.Fatalf("Config.Run() error = %v, wantErr %v", err, tt.wantErr) + err := cfg.Run(tt.args.ctx, tt.args.fn) + if diff := cmp.Diff(err, tt.wantErr, DeepAllowUnexported(RetryExhaustedError{}, errors.New(""))); diff != "" { + t.Fatalf("Config.Run() unexpected error: %s", diff) } }) } diff --git a/common/retry/utils_test.go b/common/retry/utils_test.go new file mode 100644 index 000000000..a23a7d578 --- /dev/null +++ b/common/retry/utils_test.go @@ -0,0 +1,48 @@ +package retry + +import ( + "reflect" + + "github.com/google/go-cmp/cmp" +) + +func DeepAllowUnexported(vs ...interface{}) cmp.Option { + m := make(map[reflect.Type]struct{}) + for _, v := range vs { + structTypes(reflect.ValueOf(v), m) + } + var typs []interface{} + for t := range m { + typs = append(typs, reflect.New(t).Elem().Interface()) + } + return cmp.AllowUnexported(typs...) +} + +func structTypes(v reflect.Value, m map[reflect.Type]struct{}) { + if !v.IsValid() { + return + } + switch v.Kind() { + case reflect.Ptr: + if !v.IsNil() { + structTypes(v.Elem(), m) + } + case reflect.Interface: + if !v.IsNil() { + structTypes(v.Elem(), m) + } + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + structTypes(v.Index(i), m) + } + case reflect.Map: + for _, k := range v.MapKeys() { + structTypes(v.MapIndex(k), m) + } + case reflect.Struct: + m[v.Type()] = struct{}{} + for i := 0; i < v.NumField(); i++ { + structTypes(v.Field(i), m) + } + } +}