nuageinit: Improvements for nuageinit

- Fix 'pkg update' usage:
  - The function 'nuage:run_pkg_cmd(...)' adds the flag '-y', which
    does not make sense with some commands such as 'pkg update',
    causing an error when updating the repository catalogs.
- Fix typo 'ssh-authorized-keys -> ssh_authorized_keys' in
  'nuageinit(7)' man page.
- Document 'ssh_authorized_keys' parameter.
- Use device configuration ID when no 'match' rule is specified:
  - This is the default behavior of cloud-init when no match rule is
    specified, so the device is configured anyway (even if it does not
    exist). This greatly simplifies things, since in many cases
    'if_vtnet(4)' is used, so there is no need to perform a comparison
    with the MAC address.
- Document 'network' parameter:
  - Add example to 'EXAMPLES' section.
- Set 'gateway[46]' only when 'addresses' is specified:
  - To comply with the cloud-init specification, 'gateway4' and 'gateway6'
    must only take effect when 'addresses' (or static configuration) is
    specified.
- Use a separate function to check 'match' rules:
  - This way, we can easily add new logic to new types of rules.
- Implement 'network.ethernets.{id}.match.name' parameter:
  - But unlike cloud-init, which works with glob expressions (although it
    depends on the network backend), this implementation takes advantage
    of Lua pattern-matching expressions.

    Also note that previously we were only concerned with one interface
    matching, however, to be cloud-init-compliant, we need to configure
    the matching interfaces (one or more).
- Set default router only once.
- Implement 'network.ethernets.{id}.wakeonlan' parameter.
- Implement 'network.ethernets.{id}.set-name' parameter.
- Implement 'network.ethernets.{id}.match.driver' parameter:
  - Rename 'get_ifaces(...)' function as 'get_ifaces_by_mac(...)'.
  - Add get_ifaces_by_driver(...) function.
- Implement 'network.ethernets.{id}.mtu' parameter.
- Implement 'nameservers' parameter.
- Use 'resolvconf(8)' to manipulate 'resolv.conf(5)'.
- Use 'tzsetup(8)' to set time zone.

Reviewed by:            bapt@
Approved by:            bapt@
Differential Revision:  https://reviews.freebsd.org/D51643

libexec/nuageinit/nuage.lua

@@ -451,6 +451,23 @@ local function chpasswd(obj)
 	end
 end
 
+local function settimezone(timezone)
+	if timezone == nil then
+		return
+	end
+	local root = os.getenv("NUAGE_FAKE_ROOTDIR")
+	if not root then
+		root = "/"
+	end
+
+	f, _, rc = os.execute("tzsetup -s -C " .. root .. " " .. timezone)
+
+	if not f then
+		warnmsg("Impossible to configure time zone ( rc = " .. rc .. " )")
+		return
+	end
+end
+
 local function pkg_bootstrap()
 	if os.getenv("NUAGE_RUN_TESTS") then
 		return true
@@ -480,7 +497,7 @@ local function install_package(package)
 end
 
 local function run_pkg_cmd(subcmd)
-	local cmd = "pkg " .. subcmd .. " -y"
+	local cmd = "env ASSUME_ALWAYS_YES=yes pkg " .. subcmd
 	if os.getenv("NUAGE_RUN_TESTS") then
 		print(cmd)
 		return true
@@ -556,6 +573,7 @@ local n = {
 	dirname = dirname,
 	mkdir_p = mkdir_p,
 	sethostname = sethostname,
+	settimezone = settimezone,
 	adduser = adduser,
 	addgroup = addgroup,
 	addsshkey = addsshkey,

libexec/nuageinit/nuageinit

@@ -46,7 +46,15 @@ local function open_config(name)
 	return openat("/etc/rc.conf.d", name)
 end
 
-local function get_ifaces()
+local function open_resolv_conf()
+	return openat("/etc", "resolv.conf")
+end
+
+local function open_resolvconf_conf()
+	return openat("/etc", "resolvconf.conf")
+end
+
+local function get_ifaces_by_mac()
 	local parser = ucl.parser()
 	-- grab ifaces
 	local ns = io.popen("netstat -i --libxo json")
@@ -77,6 +85,10 @@ local function sethostname(obj)
 	end
 end
 
+local function settimezone(obj)
+	nuage.settimezone(obj.timezone)
+end
+
 local function groups(obj)
 	if obj.groups == nil then return end
 
@@ -171,6 +183,59 @@ local function ssh_authorized_keys(obj)
 	end
 end
 
+local function nameservers(interface, obj)
+	local resolvconf_conf_handler = open_resolvconf_conf()
+
+	if obj.search then
+		local with_space = false
+
+		resolvconf_conf_handler:write('search_domains="')
+
+		for _, d in ipairs(obj.search) do
+			if with_space then
+				resolvconf_conf_handler:write(" " .. d)
+			else
+				resolvconf_conf_handler:write(d)
+				with_space = true
+			end
+		end
+
+		resolvconf_conf_handler:write('"\n')
+	end
+
+	if obj.addresses then
+		local with_space = false
+
+		resolvconf_conf_handler:write('name_servers="')
+
+		for _, a in ipairs(obj.addresses) do
+			if with_space then
+				resolvconf_conf_handler:write(" " .. a)
+			else
+				resolvconf_conf_handler:write(a)
+				with_space = true
+			end
+		end
+
+		resolvconf_conf_handler:write('"\n')
+	end
+
+	resolvconf_conf_handler:close()
+
+	local resolv_conf = root .. "/etc/resolv.conf"
+
+	resolv_conf_attr = lfs.attributes(resolv_conf)
+
+	if resolv_conf_attr == nil then
+		resolv_conf_handler = open_resolv_conf()
+		resolv_conf_handler:close()
+	end
+
+	if not os.execute("resolvconf -a " .. interface .. " < " .. resolv_conf) then
+		nuage.warn("Failed to execute resolvconf(8)")
+	end
+end
+
 local function install_packages(packages)
 	if not nuage.pkg_bootstrap() then
 		nuage.warn("Failed to bootstrap pkg, skip installing packages")
@@ -187,6 +252,85 @@ local function install_packages(packages)
 	end
 end
 
+local function list_ifaces()
+	local proc = io.popen("ifconfig -l")
+	local raw_ifaces = proc:read("*a")
+	proc:close()
+	local ifaces = {}
+	for i in raw_ifaces:gmatch("[^%s]+") do
+		table.insert(ifaces, i)
+	end
+	return ifaces
+end
+
+local function get_ifaces_by_driver()
+	local proc = io.popen("ifconfig -D")
+	local drivers = {}
+	local last_interface = nil
+	for line in proc:lines() do
+	    local interface = line:match("^([%S]+): ")
+
+	    if interface then
+		last_interface = interface
+	    end
+
+	    local driver = line:match("^[%s]+drivername: ([%S]+)$")
+
+	    if driver then
+		drivers[driver] = last_interface
+	    end
+	end
+	proc:close()
+
+	return drivers
+end
+
+local function match_rules(rules)
+	-- To comply with the cloud-init specification, all rules must match and a table
+	-- with the matching interfaces must be returned. This changes the way we initially
+	-- thought about our implementation, since at first we only needed one interface,
+	-- but cloud-init performs actions on a group of matching interfaces.
+	local interfaces = {}
+	if rules.macaddress then
+		local ifaces = get_ifaces_by_mac()
+		local interface = ifaces[rules.macaddress]
+		if not interface then
+			nuage.warn("not interface matching by MAC address: " .. rules.macaddress)
+			return
+		end
+		interfaces[interface] = 1
+	end
+	if rules.name then
+		local match = false
+		for _, i in pairs(list_ifaces()) do
+			if i:match(rules.name) then
+				match = true
+				interfaces[i] = 1
+			end
+		end
+		if not match then
+			nuage.warn("not interface matching by name: " .. rules.name)
+			return
+		end
+	end
+	if rules.driver then
+		local match = false
+		local drivers = get_ifaces_by_driver()
+		for d in pairs(drivers) do
+			if d:match(rules.driver) then
+				match = true
+				interface = drivers[d]
+				interfaces[interface] = 1
+			end
+		end
+		if not match then
+			nuage.warn("not interface matching by driver: " .. rules.driver)
+			return
+		end
+	end
+	return interfaces
+end
+
 local function write_files(files, defer)
 	if not files then
 		return
@@ -210,41 +354,76 @@ end
 local function network_config(obj)
 	if obj.network == nil then return end
 
-	local ifaces = get_ifaces()
 	local network = open_config("network")
 	local routing = open_config("routing")
 	local ipv6 = {}
-	for _, v in pairs(obj.network.ethernets) do
-		if not v.match then
-			goto next
+	local set_defaultrouter = true
+	local set_defaultrouter6 = true
+	local set_nameservers = true
+	for i, v in pairs(obj.network.ethernets) do
+		local interfaces = {}
+		if v.match then
+			interfaces = match_rules(v.match)
+
+			if next(interfaces) == nil then
+				goto next
+			end
+		else
+			interfaces[i] = 1
 		end
-		if not v.match.macaddress then
-			goto next
+		local extra_opts = ""
+		if v.wakeonlan then
+			extra_opts = extra_opts .. " wol"
 		end
-		if not ifaces[v.match.macaddress] then
-			nuage.warn("not interface matching: " .. v.match.macaddress)
-			goto next
-		end
-		local interface = ifaces[v.match.macaddress]
-		if v.dhcp4 then
-			network:write("ifconfig_" .. interface .. '="DHCP"\n')
-		elseif v.addresses then
-			for _, a in pairs(v.addresses) do
-				if a:match("^(%d+)%.(%d+)%.(%d+)%.(%d+)") then
-					network:write("ifconfig_" .. interface .. '="inet ' .. a .. '"\n')
-				else
-					network:write("ifconfig_" .. interface .. '_ipv6="inet6 ' .. a .. '"\n')
-					ipv6[#ipv6 + 1] = interface
-				end
+		if v.mtu then
+			if type(v.mtu) == "number" then
+				mtu = tostring(v.mtu)
+			else
+				mtu = v.mtu
+			end
+			if mtu:match("%d") then
+				extra_opts = extra_opts .. " mtu " .. mtu
+			else
+				nuage.warn("MTU is not set because the specified value is invalid: " .. mtu)
 			end
 		end
-		if v.gateway4 then
-			routing:write('defaultrouter="' .. v.gateway4 .. '"\n')
-		end
-		if v.gateway6 then
-			routing:write('ipv6_defaultrouter="' .. v.gateway6 .. '"\n')
-			routing:write("ipv6_route_" .. interface .. '="' .. v.gateway6)
-			routing:write(" -prefixlen 128 -interface " .. interface .. '"\n')
+		for interface in pairs(interfaces) do
+			if v.match and v.match.macaddress and v["set-name"] then
+				local ifaces = get_ifaces_by_mac()
+				local matched = ifaces[v.match.macaddress]
+				if matched and matched == interface then
+					network:write("ifconfig_" .. interface .. '_name=' .. v["set-name"] .. '\n')
+					interface = v["set-name"]
+				end
+			end
+			if v.dhcp4 then
+				network:write("ifconfig_" .. interface .. '="DHCP"' .. extra_opts .. '\n')
+			elseif v.addresses then
+				for _, a in pairs(v.addresses) do
+					if a:match("^(%d+)%.(%d+)%.(%d+)%.(%d+)") then
+						network:write("ifconfig_" .. interface .. '="inet ' .. a .. extra_opts .. '"\n')
+					else
+						network:write("ifconfig_" .. interface .. '_ipv6="inet6 ' .. a .. extra_opts .. '"\n')
+						ipv6[#ipv6 + 1] = interface
+					end
+				end
+				if set_nameservers and v.nameservers then
+					set_nameservers = false
+					nameservers(interface, v.nameservers)
+				end
+				if set_defaultrouter and v.gateway4 then
+					set_defaultrouter = false
+					routing:write('defaultrouter="' .. v.gateway4 .. '"\n')
+				end
+				if v.gateway6 then
+					if set_defaultrouter6 then
+						set_defaultrouter6 = false
+						routing:write('ipv6_defaultrouter="' .. v.gateway6 .. '"\n')
+					end
+					routing:write("ipv6_route_" .. interface .. '="' .. v.gateway6)
+					routing:write(" -prefixlen 128 -interface " .. interface .. '"\n')
+				end
+			end
 		end
 		::next::
 	end
@@ -316,7 +495,7 @@ local function config2_network(p)
 	end
 	local obj = parser:get_object()
 
-	local ifaces = get_ifaces()
+	local ifaces = get_ifaces_by_mac()
 	if not ifaces then
 		nuage.warn("no network interfaces found")
 		return
@@ -468,6 +647,7 @@ f:close()
 if line == "#cloud-config" then
 	local pre_network_calls = {
 		sethostname,
+		settimezone,
 		groups,
 		create_default_user,
 		ssh_keys,

libexec/nuageinit/nuageinit.7

@@ -143,6 +143,11 @@ Specify a fully qualified domain name for the instance.
 Specify the hostname of the instance if
 .Qq Ic fqdn
 is not set.
+.It Ic timezone
+Sets the system timezone based on the value provided.
+.Pp
+See also
+.Xr tzfile 3 Ns .
 .It Ic groups
 An array of strings or objects to be created:
 .Bl -bullet
@@ -176,6 +181,81 @@ boolean which determines the value of the
 configuration in
 .Pa /etc/ssh/sshd_config
 .It Ic network
+Network configuration parameters.
+.Bl -tag -width "ethernets"
+.It Ic ethernets
+Mapping representing a generic configuration for existing network interfaces.
+.Pp
+Each key is an interface name that is only used when no
+.Sy match
+rule is specified.
+If
+.Sy match
+rules are specified, an arbitrary name can be used
+.Po e.g.: id0 Pc Ns .
+.Bl -tag -width "nameservers"
+.It Ic match
+This selects a subset of available physical devices by various hardware properties.
+The following configuration will then apply to all matching devices, as soon as
+they appear. All specified properties must match. The following properties for
+creating matches are supported:
+.Bl -tag -width "macaddress"
+.It Ic macaddress
+.No Device's MAC address in the form Sy xx:xx:xx:xx:xx:xx Ns .
+Letters should be lowercase.
+.It Ic name
+Current interface name. Lua pattern-matching expressions are supported.
+.It Ic driver
+Interface driver name and unit number of the interface. Lua pattern-natching expressions
+are supported.
+.El
+.It Ic set-name
+When matching on unique properties such as MAC, match rules can be written so that they
+match only one device. Then this property can be used to give that device a more
+specific/desirable/nicer name than the default.
+.Pp
+While multiple properties can be used in a match,
+.Sy macaddress
+is required for nuageinit to perform the rename.
+.It Ic mtu
+The MTU key represents a device's Maximum Transmission Unit, the largest size packet
+or frame.
+.It Ic wakeonlan
+Enable wake on LAN. Off by default.
+.It Ic dhcp4
+Configure the interface to use DHCP.
+.Pp
+This takes precedence over
+.Sy addresses
+when both are specified.
+.It Ic addresses
+List of strings representing IPv4 or IPv6 addresses.
+.It Ic gateway4
+Set default gateway for IPv4, for manual address configuration. This requires setting
+.Sy addresses
+too.
+.Pp
+Since only one default router can be configured at a time, this parameter is applied
+when processing the first entry, and any others are silently ignored.
+.It Ic gateway6
+Set default gateway for IPv6, for manual address configuration. This requires setting
+.Sy addresses
+too.
+.Pp
+Since only one default router can be configured at a time, this parameter is applied
+when processing the first entry, and any others are silently ignored.
+.It Ic nameservers
+Set DNS servers and search domains, for manual address configuration.
+.Pp
+There are two supported fields:
+.Bl -tag -width "addresses"
+.It Ic search
+Search list for host-name lookup.
+.It Ic addresses
+List of IPv4 or IPv6 name server addresses that the resolver should query.
+.El
+.El
+.El
 .It Ic runcmd
 An array of commands to be run at the end of the boot process
 .It Ic packages
@@ -186,7 +266,7 @@ Update the remote package metadata.
 Upgrade the packages installed to their latest version.
 .It Ic users
 Specify a list of users to be created:
-.Bl -tag -width "plain_text_passwd"
+.Bl -tag -width "ssh_authorized_keys"
 .It Ic name
 Name of the user.
 .It Ic gecos
@@ -201,6 +281,8 @@ The list of other groups the user should belong to.
 A boolean which determines if the home directory should be created or not.
 .It Ic shell
 The shell that should be used for the user.
+.It Ic ssh_authorized_keys
+List of SSH keys for the user.
 .It Ic passwd
 The encrypted password for the user.
 .It Ic plain_text_passwd
@@ -287,7 +369,7 @@ users:
   - name: user
     gecos: Foo B. Bar
     sudo: ALL=(ALL) NOPASSWD:ALL
-    ssh-authorized-keys:
+    ssh_authorized_keys:
       - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAr...
 packages:
   - neovim
@@ -303,6 +385,12 @@ ssh_keys:
     ...
     -----END OPENSSH PRIVATE KEY-----
   ed25519_public: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK+MH4E8KO32N5CXRvXVqvyZVl0+6ue4DobdhU0FqFd+
+network:
+  ethernets:
+    vtnet0:
+      addresses:
+        - 192.168.8.2/24
+      gateway4: 192.168.8.1
 .Ed
 .Sh SEE ALSO
 .Xr kenv 2 ,

libexec/nuageinit/tests/Makefile

@@ -15,6 +15,7 @@ ${PACKAGE}FILES+=	adduser_passwd.lua
 ${PACKAGE}FILES+=	dirname.lua
 ${PACKAGE}FILES+=	err.lua
 ${PACKAGE}FILES+=	sethostname.lua
+${PACKAGE}FILES+=	settimezone.lua
 ${PACKAGE}FILES+=	warn.lua
 ${PACKAGE}FILES+=	addfile.lua
 

libexec/nuageinit/tests/nuage.sh

@@ -7,12 +7,21 @@
 export NUAGE_FAKE_ROOTDIR="$PWD"
 
 atf_test_case sethostname
+atf_test_case settimezone
 atf_test_case addsshkey
 atf_test_case adduser
 atf_test_case adduser_passwd
 atf_test_case addgroup
 atf_test_case addfile
 
+settimezone_body()
+{
+	atf_check /usr/libexec/flua $(atf_get_srcdir)/settimezone.lua
+	if [ ! -f etc/localtime ]; then
+		atf_fail "localtime not written"
+	fi
+}
+
 sethostname_body()
 {
 	atf_check /usr/libexec/flua $(atf_get_srcdir)/sethostname.lua

libexec/nuageinit/tests/nuageinit.sh

@@ -815,7 +815,7 @@ config2_userdata_update_packages_body()
 package_update: true
 EOF
 	chmod 755 "${PWD}"/media/nuageinit/user_data
-	atf_check -o inline:"pkg update -y\n" /usr/libexec/nuageinit "${PWD}"/media/nuageinit postnet
+	atf_check -o inline:"env ASSUME_ALWAYS_YES=yes pkg update\n" /usr/libexec/nuageinit "${PWD}"/media/nuageinit postnet
 }
 
 config2_userdata_upgrade_packages_body()
@@ -829,7 +829,7 @@ config2_userdata_upgrade_packages_body()
 package_upgrade: true
 EOF
 	chmod 755 "${PWD}"/media/nuageinit/user_data
-	atf_check -o inline:"pkg upgrade -y\n" /usr/libexec/nuageinit "${PWD}"/media/nuageinit postnet
+	atf_check -o inline:"env ASSUME_ALWAYS_YES=yes pkg upgrade\n" /usr/libexec/nuageinit "${PWD}"/media/nuageinit postnet
 }
 
 config2_userdata_shebang_body()

libexec/nuageinit/tests/settimezone.lua

@@ -0,0 +1,5 @@
+#!/usr/libexec/flua
+
+local n = require("nuage")
+
+n.settimezone("UTC")