From efcf8eb7acd0220010582afeffd9de45e788c610 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 5 Dec 2025 22:51:05 +0000 Subject: [PATCH] Move embedded flannel and vpn config setup into embedded executor Flannel and VPN setup shouldn't be done in generic agent config as it is only used with embeded executor's flannel CNI. Signed-off-by: Brad Davidson --- pkg/agent/config/config.go | 75 ---------------------------- pkg/daemons/config/types.go | 1 - pkg/daemons/executor/embed.go | 94 ++++++++++++++++++++++++++++++++++- 3 files changed, 93 insertions(+), 77 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 537f47639fd..eac7ab1c082 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -16,7 +16,6 @@ import ( "net/http" "net/url" "os" - "os/exec" "path/filepath" "regexp" "strconv" @@ -32,7 +31,6 @@ import ( "github.com/k3s-io/k3s/pkg/spegel" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" - "github.com/k3s-io/k3s/pkg/vpn" pkgerrors "github.com/pkg/errors" certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/wharfie/pkg/registries" @@ -470,14 +468,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } apiServerURL := proxy.APIServerURL() - var flannelIface *net.Interface - if controlConfig.FlannelBackend != config.FlannelBackendNone && len(envInfo.FlannelIface) > 0 { - flannelIface, err = net.InterfaceByName(envInfo.FlannelIface) - if err != nil { - return nil, pkgerrors.WithMessagef(err, "unable to find interface %s", envInfo.FlannelIface) - } - } - clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt") if err := getHostFile(clientCAFile, info); err != nil { return nil, err @@ -504,47 +494,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N newNodePasswordFile := filepath.Join(nodeConfigPath, "password") upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile) - // If there is a VPN, we must overwrite NodeIP and flannel interface - var vpnInfo vpn.VPNInfo - if envInfo.VPNAuth != "" { - vpnInfo, err = vpn.GetVPNInfo(envInfo.VPNAuth) - if err != nil { - return nil, err - } - - // Pass ipv4, ipv6 or both depending on nodeIPs mode - var vpnIPs []net.IP - if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv4Address) - if vpnInfo.IPv6Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv6Address) - } - } else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv6Address) - if vpnInfo.IPv4Address != nil { - vpnIPs = append(vpnIPs, vpnInfo.IPv4Address) - } - } else { - return nil, fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address) - } - - // Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters - if len(vpnIPs) != 0 { - logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs) - if len(envInfo.NodeIP.Value()) != 0 { - logrus.Warn("VPN provider overrides configured node-ip parameter") - } - if len(envInfo.NodeExternalIP.Value()) != 0 { - logrus.Warn("VPN provider overrides node-external-ip parameter") - } - nodeIPs = vpnIPs - flannelIface, err = net.InterfaceByName(vpnInfo.VPNInterface) - if err != nil { - return nil, pkgerrors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface) - } - } - } - if controlConfig.ClusterIPRange != nil { if utilsnet.IPFamilyOfCIDR(controlConfig.ClusterIPRange) != utilsnet.IPFamilyOf(nodeIPs[0]) && len(nodeIPs) > 1 { firstNodeIP := nodeIPs[0] @@ -639,7 +588,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N SupervisorMetrics: controlConfig.SupervisorMetrics, Token: info.String(), } - nodeConfig.FlannelIface = flannelIface nodeConfig.Images = filepath.Join(envInfo.DataDir, "agent", "images") nodeConfig.AgentConfig.NodeName = nodeName nodeConfig.AgentConfig.NodeConfigPath = nodeConfigPath @@ -705,29 +653,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } nodeConfig.AgentConfig.NodeInternalDNSs = nodeInternalDNSs - nodeConfig.NoFlannel = nodeConfig.FlannelBackend == config.FlannelBackendNone - if !nodeConfig.NoFlannel { - hostLocal, err := exec.LookPath("host-local") - if err != nil { - return nil, pkgerrors.WithMessagef(err, "failed to find host-local") - } - - if envInfo.FlannelConf == "" { - nodeConfig.FlannelConfFile = filepath.Join(envInfo.DataDir, "agent", "etc", "flannel", "net-conf.json") - } else { - nodeConfig.FlannelConfFile = envInfo.FlannelConf - nodeConfig.FlannelConfOverride = true - } - nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal) - nodeConfig.AgentConfig.CNIConfDir = filepath.Join(envInfo.DataDir, "agent", "etc", "cni", "net.d") - nodeConfig.AgentConfig.FlannelCniConfFile = envInfo.FlannelCniConfFile - - // It does not make sense to use VPN without its flannel backend - if envInfo.VPNAuth != "" { - nodeConfig.FlannelBackend = vpnInfo.ProviderName - } - } - if nodeConfig.ImageServiceEndpoint != "" { nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 3a395e54edf..ea998317ef8 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -40,7 +40,6 @@ type Node struct { Docker bool ContainerRuntimeEndpoint string ImageServiceEndpoint string - NoFlannel bool SELinux bool EnablePProf bool SupervisorMetrics bool diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index f8f3e7dc11b..b768627aed7 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -7,7 +7,11 @@ import ( "context" "errors" "flag" + "fmt" + "net" "net/http" + "os/exec" + "path/filepath" "runtime/debug" "strconv" "sync" @@ -19,10 +23,12 @@ import ( "github.com/k3s-io/k3s/pkg/agent/flannel" "github.com/k3s-io/k3s/pkg/agent/netpol" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/daemons/config" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/signals" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" + "github.com/k3s-io/k3s/pkg/vpn" pkgerrors "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -38,6 +44,7 @@ import ( proxy "k8s.io/kubernetes/cmd/kube-proxy/app" sapp "k8s.io/kubernetes/cmd/kube-scheduler/app" kubelet "k8s.io/kubernetes/cmd/kubelet/app" + utilsnet "k8s.io/utils/net" // registering k3s cloud provider _ "github.com/k3s-io/k3s/pkg/cloudprovider" @@ -74,6 +81,82 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, } }) + if nodeConfig.FlannelBackend != config.FlannelBackendNone { + var err error + + var flannelIface *net.Interface + if len(cfg.FlannelIface) > 0 { + flannelIface, err = net.InterfaceByName(cfg.FlannelIface) + if err != nil { + return pkgerrors.WithMessagef(err, "unable to find interface %s", cfg.FlannelIface) + } + } + nodeConfig.FlannelIface = flannelIface + + // If there is a VPN, we must overwrite NodeIP and flannel interface + var vpnInfo vpn.VPNInfo + if cfg.VPNAuth != "" { + vpnInfo, err = vpn.GetVPNInfo(cfg.VPNAuth) + if err != nil { + return err + } + + // Pass ipv4, ipv6 or both depending on nodeIPs mode + nodeIPs := nodeConfig.AgentConfig.NodeIPs + var vpnIPs []net.IP + if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil { + vpnIPs = append(vpnIPs, vpnInfo.IPv4Address) + if vpnInfo.IPv6Address != nil { + vpnIPs = append(vpnIPs, vpnInfo.IPv6Address) + } + } else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil { + vpnIPs = append(vpnIPs, vpnInfo.IPv6Address) + if vpnInfo.IPv4Address != nil { + vpnIPs = append(vpnIPs, vpnInfo.IPv4Address) + } + } else { + return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address) + } + + // Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters + if len(vpnIPs) != 0 { + logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs) + if len(cfg.NodeIP.Value()) != 0 { + logrus.Warn("VPN provider overrides configured node-ip parameter") + } + if len(cfg.NodeExternalIP.Value()) != 0 { + logrus.Warn("VPN provider overrides node-external-ip parameter") + } + nodeIPs = vpnIPs + flannelIface, err = net.InterfaceByName(vpnInfo.VPNInterface) + if err != nil { + return pkgerrors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface) + } + } + } + + // set paths for embedded flannel if enabled + hostLocal, err := exec.LookPath("host-local") + if err != nil { + return pkgerrors.WithMessagef(err, "failed to find host-local") + } + + if cfg.FlannelConf == "" { + nodeConfig.FlannelConfFile = filepath.Join(cfg.DataDir, "agent", "etc", "flannel", "net-conf.json") + } else { + nodeConfig.FlannelConfFile = cfg.FlannelConf + nodeConfig.FlannelConfOverride = true + } + nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal) + nodeConfig.AgentConfig.CNIConfDir = filepath.Join(cfg.DataDir, "agent", "etc", "cni", "net.d") + nodeConfig.AgentConfig.FlannelCniConfFile = cfg.FlannelCniConfFile + + // It does not make sense to use VPN without its flannel backend + if cfg.VPNAuth != "" { + nodeConfig.FlannelBackend = vpnInfo.ProviderName + } + } + return nil } @@ -254,7 +337,16 @@ func (e *Embedded) CRI(ctx context.Context, cfg *daemonconfig.Node) error { } func (e *Embedded) CNI(ctx context.Context, wg *sync.WaitGroup, cfg *daemonconfig.Node) error { - if !cfg.NoFlannel { + if cfg.FlannelBackend != daemonconfig.FlannelBackendNone { + if (cfg.FlannelExternalIP) && (len(cfg.AgentConfig.NodeExternalIPs) == 0) { + logrus.Warnf("Server has flannel-external-ip flag set but this node does not set node-external-ip. Flannel will use internal address when connecting to this node.") + } else if (cfg.FlannelExternalIP) && (cfg.FlannelBackend != daemonconfig.FlannelBackendWireguardNative) { + logrus.Warnf("Flannel is using external addresses with an insecure backend: %v. Please consider using an encrypting flannel backend.", cfg.FlannelBackend) + } + if err := flannel.Prepare(ctx, cfg); err != nil { + return err + } + if err := flannel.Run(ctx, wg, cfg); err != nil { return err }