diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index eac7ab1c082..7268dfdc9d4 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -579,14 +579,16 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N ImageServiceEndpoint: envInfo.ImageServiceEndpoint, EnablePProf: envInfo.EnablePProf, EmbeddedRegistry: controlConfig.EmbeddedRegistry, - FlannelBackend: controlConfig.FlannelBackend, - FlannelIPv6Masq: controlConfig.FlannelIPv6Masq, - FlannelExternalIP: controlConfig.FlannelExternalIP, EgressSelectorMode: controlConfig.EgressSelectorMode, ServerHTTPSPort: controlConfig.HTTPSPort, SupervisorPort: controlConfig.SupervisorPort, SupervisorMetrics: controlConfig.SupervisorMetrics, Token: info.String(), + Flannel: config.Flannel{ + Backend: controlConfig.FlannelBackend, + IPv6Masq: controlConfig.FlannelIPv6Masq, + ExternalIP: controlConfig.FlannelExternalIP, + }, } nodeConfig.Images = filepath.Join(envInfo.DataDir, "agent", "images") nodeConfig.AgentConfig.NodeName = nodeName diff --git a/pkg/agent/flannel/flannel.go b/pkg/agent/flannel/flannel.go index 1b4d11f08f2..0c6cd311da8 100644 --- a/pkg/agent/flannel/flannel.go +++ b/pkg/agent/flannel/flannel.go @@ -43,12 +43,18 @@ import ( const ( subnetFile = "/run/flannel/subnet.env" + + BackendNone = "none" + BackendVXLAN = "vxlan" + BackendHostGW = "host-gw" + BackendWireguardNative = "wireguard-native" + BackendTailscale = "tailscale" ) var ( - FlannelBaseAnnotation = "flannel.alpha.coreos.com" - FlannelExternalIPv4Annotation = FlannelBaseAnnotation + "/public-ip-overwrite" - FlannelExternalIPv6Annotation = FlannelBaseAnnotation + "/public-ipv6-overwrite" + BaseAnnotation = "flannel.alpha.coreos.com" + ExternalIPv4Annotation = BaseAnnotation + "/public-ip-overwrite" + ExternalIPv6Annotation = BaseAnnotation + "/public-ipv6-overwrite" ) func flannel(ctx context.Context, wg *sync.WaitGroup, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv6Masq bool, nm netMode) error { @@ -60,7 +66,7 @@ func flannel(ctx context.Context, wg *sync.WaitGroup, flannelIface *net.Interfac sm, err := kube.NewSubnetManager(ctx, "", kubeConfigFile, - FlannelBaseAnnotation, + BaseAnnotation, flannelConf, false) if err != nil { diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index 0c114e47063..b2a16bba990 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -75,7 +75,7 @@ func Prepare(ctx context.Context, nodeConfig *config.Node) error { } func Run(ctx context.Context, wg *sync.WaitGroup, nodeConfig *config.Node) error { - logrus.Infof("Starting flannel with backend %s", nodeConfig.FlannelBackend) + logrus.Infof("Starting flannel with backend %s", nodeConfig.Flannel.Backend) kubeConfig := nodeConfig.AgentConfig.KubeConfigKubelet coreClient, err := util.GetClientSet(kubeConfig) @@ -116,7 +116,7 @@ func Run(ctx context.Context, wg *sync.WaitGroup, nodeConfig *config.Node) error return pkgerrors.WithMessage(err, "failed to check netMode for flannel") } go func() { - err := flannel(ctx, wg, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv6Masq, nm) + err := flannel(ctx, wg, nodeConfig.Flannel.Iface, nodeConfig.Flannel.ConfFile, kubeConfig, nodeConfig.Flannel.IPv6Masq, nm) if err != nil && !errors.Is(err, context.Canceled) { signals.RequestShutdown(pkgerrors.WithMessage(err, "flannel exited")) } @@ -151,14 +151,14 @@ func createCNIConf(dir string, nodeConfig *config.Node) error { } p := filepath.Join(dir, "10-flannel.conflist") - if nodeConfig.AgentConfig.FlannelCniConfFile != "" { - logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile) - return agentutil.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false) + if nodeConfig.Flannel.CNIConfFile != "" { + logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.Flannel.CNIConfFile) + return agentutil.CopyFile(nodeConfig.Flannel.CNIConfFile, p, false) } cniConfJSON := cniConf if goruntime.GOOS == "windows" { - extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4) + extIface, err := LookupExtInterface(nodeConfig.Flannel.Iface, ipv4) if err != nil { return err } @@ -172,12 +172,12 @@ func createCNIConf(dir string, nodeConfig *config.Node) error { } func createFlannelConf(nodeConfig *config.Node) error { - logrus.Debugf("Creating the flannel configuration for backend %s in file %s", nodeConfig.FlannelBackend, nodeConfig.FlannelConfFile) - if nodeConfig.FlannelConfFile == "" { + logrus.Debugf("Creating the flannel configuration for backend %s in file %s", nodeConfig.Flannel.Backend, nodeConfig.Flannel.ConfFile) + if nodeConfig.Flannel.ConfFile == "" { return errors.New("Flannel configuration not defined") } - if nodeConfig.FlannelConfOverride { - logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConfFile) + if nodeConfig.Flannel.ConfOverride { + logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.Flannel.ConfFile) return nil } nm, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs) @@ -219,21 +219,21 @@ func createFlannelConf(nodeConfig *config.Node) error { var backendConf string // precheck and error out unsupported flannel backends. - switch nodeConfig.FlannelBackend { - case config.FlannelBackendHostGW: - case config.FlannelBackendTailscale: - case config.FlannelBackendWireguardNative: + switch nodeConfig.Flannel.Backend { + case BackendHostGW: + case BackendTailscale: + case BackendWireguardNative: if goruntime.GOOS == "windows" { - return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend) + return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.Flannel.Backend) } } - switch nodeConfig.FlannelBackend { - case config.FlannelBackendVXLAN: + switch nodeConfig.Flannel.Backend { + case BackendVXLAN: backendConf = vxlanBackend - case config.FlannelBackendHostGW: + case BackendHostGW: backendConf = hostGWBackend - case config.FlannelBackendTailscale: + case BackendTailscale: var routes []string if nm.IPv4Enabled() { routes = append(routes, "$SUBNET") @@ -251,15 +251,15 @@ func createFlannelConf(nodeConfig *config.Node) error { } } backendConf = strings.ReplaceAll(tailscaledBackend, "%Routes%", strings.Join(routes, ",")) - case config.FlannelBackendWireguardNative: + case BackendWireguardNative: backendConf = wireguardNativeBackend default: - return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend) + return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.Flannel.Backend) } confJSON = strings.ReplaceAll(confJSON, "%backend%", backendConf) logrus.Debugf("The flannel configuration is %s", confJSON) - return agentutil.WriteFile(nodeConfig.FlannelConfFile, confJSON) + return agentutil.WriteFile(nodeConfig.Flannel.ConfFile, confJSON) } // fundNetMode returns the mode (ipv4, ipv6 or dual-stack) in which flannel is operating @@ -286,13 +286,13 @@ func findNetMode(cidrs []*net.IPNet) (netMode, error) { func setAnnotations(ctx context.Context, nodeConfig *config.Node, coreClient kubernetes.Interface) error { patch := util.NewPatchList() patcher := util.NewPatcher[*v1.Node](coreClient.CoreV1().Nodes()) - if nodeConfig.AgentConfig.NodeExternalIP != "" && nodeConfig.FlannelExternalIP { + if nodeConfig.AgentConfig.NodeExternalIP != "" && nodeConfig.Flannel.ExternalIP { for _, ipAddress := range nodeConfig.AgentConfig.NodeExternalIPs { if utilsnet.IsIPv4(ipAddress) { - patch.Add(ipAddress.String(), "metadata", "annotations", FlannelExternalIPv4Annotation) + patch.Add(ipAddress.String(), "metadata", "annotations", ExternalIPv4Annotation) } if utilsnet.IsIPv6(ipAddress) { - patch.Add(ipAddress.String(), "metadata", "annotations", FlannelExternalIPv6Annotation) + patch.Add(ipAddress.String(), "metadata", "annotations", ExternalIPv6Annotation) } } } diff --git a/pkg/agent/flannel/setup_test.go b/pkg/agent/flannel/setup_test.go index 2c712f09f53..20450deb8cb 100644 --- a/pkg/agent/flannel/setup_test.go +++ b/pkg/agent/flannel/setup_test.go @@ -62,12 +62,17 @@ func Test_createFlannelConf(t *testing.T) { {"dual-stack", "10.42.0.0/16,2001:cafe:22::/56", []string{"\"Network\": \"10.42.0.0/16\"", "\"IPv6Network\": \"2001:cafe:22::/56\"", "\"EnableIPv6\": true"}, false}, {"ipv4 only", "10.42.0.0/16", []string{"\"Network\": \"10.42.0.0/16\"", "\"IPv6Network\": \"::/0\"", "\"EnableIPv6\": false"}, false}, } - var containerd = config.Containerd{} for _, tt := range tests { - var agent = config.Agent{} - agent.ClusterCIDR = stringToCIDR(tt.args)[0] - agent.ClusterCIDRs = stringToCIDR(tt.args) - var nodeConfig = &config.Node{Docker: false, ContainerRuntimeEndpoint: "", SELinux: false, FlannelBackend: "vxlan", FlannelConfFile: "test_file", FlannelConfOverride: false, FlannelIface: nil, Containerd: containerd, Images: "", AgentConfig: agent, Token: "", ServerHTTPSPort: 0} + var nodeConfig = &config.Node{ + Flannel: config.Flannel{ + Backend: "vxlan", + ConfFile: "test_file", + }, + AgentConfig: config.Agent{ + ClusterCIDR: stringToCIDR(tt.args)[0], + ClusterCIDRs: stringToCIDR(tt.args), + }, + } t.Run(tt.name, func(t *testing.T) { if err := createFlannelConf(nodeConfig); (err != nil) != tt.wantErr { diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index ea998317ef8..64ea79190cd 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -23,17 +23,12 @@ import ( ) const ( - FlannelBackendNone = "none" - FlannelBackendVXLAN = "vxlan" - FlannelBackendHostGW = "host-gw" - FlannelBackendWireguardNative = "wireguard-native" - FlannelBackendTailscale = "tailscale" - EgressSelectorModeAgent = "agent" - EgressSelectorModeCluster = "cluster" - EgressSelectorModeDisabled = "disabled" - EgressSelectorModePod = "pod" - CertificateRenewDays = 120 - StreamServerPort = "10010" + EgressSelectorModeAgent = "agent" + EgressSelectorModeCluster = "cluster" + EgressSelectorModeDisabled = "disabled" + EgressSelectorModePod = "pod" + CertificateRenewDays = 120 + StreamServerPort = "10010" ) type Node struct { @@ -44,15 +39,10 @@ type Node struct { EnablePProf bool SupervisorMetrics bool EmbeddedRegistry bool - FlannelBackend string - FlannelConfFile string - FlannelConfOverride bool - FlannelIface *net.Interface - FlannelIPv6Masq bool - FlannelExternalIP bool EgressSelectorMode string Containerd Containerd CRIDockerd CRIDockerd + Flannel Flannel Images string AgentConfig Agent Token string @@ -103,6 +93,16 @@ type CRIDockerd struct { Debug bool } +type Flannel struct { + Backend string + CNIConfFile string + ConfFile string + ConfOverride bool + Iface *net.Interface + IPv6Masq bool + ExternalIP bool +} + type Agent struct { PodManifests string NodeName string @@ -148,7 +148,6 @@ type Agent struct { ImageCredProvBinDir string ImageCredProvConfig string IPSECPSK string - FlannelCniConfFile string Registry *registries.Registry SystemDefaultRegistry string AirgapExtraRegistry []string diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index b768627aed7..0c7f39cfecd 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -23,7 +23,6 @@ import ( "github.com/k3s-io/k3s/pkg/agent/flannel" "github.com/k3s-io/k3s/pkg/agent/netpol" "github.com/k3s-io/k3s/pkg/cli/cmds" - "github.com/k3s-io/k3s/pkg/daemons/config" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/signals" "github.com/k3s-io/k3s/pkg/util" @@ -81,17 +80,15 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, } }) - if nodeConfig.FlannelBackend != config.FlannelBackendNone { + if nodeConfig.Flannel.Backend != flannel.BackendNone { var err error - var flannelIface *net.Interface if len(cfg.FlannelIface) > 0 { - flannelIface, err = net.InterfaceByName(cfg.FlannelIface) + nodeConfig.Flannel.Iface, err = net.InterfaceByName(cfg.FlannelIface) if err != nil { return pkgerrors.WithMessagef(err, "unable to find interface %s", cfg.FlannelIface) } } - nodeConfig.FlannelIface = flannelIface // If there is a VPN, we must overwrite NodeIP and flannel interface var vpnInfo vpn.VPNInfo @@ -128,7 +125,7 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, logrus.Warn("VPN provider overrides node-external-ip parameter") } nodeIPs = vpnIPs - flannelIface, err = net.InterfaceByName(vpnInfo.VPNInterface) + nodeConfig.Flannel.Iface, err = net.InterfaceByName(vpnInfo.VPNInterface) if err != nil { return pkgerrors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface) } @@ -142,18 +139,17 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, } if cfg.FlannelConf == "" { - nodeConfig.FlannelConfFile = filepath.Join(cfg.DataDir, "agent", "etc", "flannel", "net-conf.json") + nodeConfig.Flannel.ConfFile = filepath.Join(cfg.DataDir, "agent", "etc", "flannel", "net-conf.json") } else { - nodeConfig.FlannelConfFile = cfg.FlannelConf - nodeConfig.FlannelConfOverride = true + nodeConfig.Flannel.ConfFile = cfg.FlannelConf + nodeConfig.Flannel.ConfOverride = true } nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal) nodeConfig.AgentConfig.CNIConfDir = filepath.Join(cfg.DataDir, "agent", "etc", "cni", "net.d") - nodeConfig.AgentConfig.FlannelCniConfFile = cfg.FlannelCniConfFile // It does not make sense to use VPN without its flannel backend if cfg.VPNAuth != "" { - nodeConfig.FlannelBackend = vpnInfo.ProviderName + nodeConfig.Flannel.Backend = vpnInfo.ProviderName } } @@ -337,11 +333,11 @@ func (e *Embedded) CRI(ctx context.Context, cfg *daemonconfig.Node) error { } func (e *Embedded) CNI(ctx context.Context, wg *sync.WaitGroup, cfg *daemonconfig.Node) error { - if cfg.FlannelBackend != daemonconfig.FlannelBackendNone { - if (cfg.FlannelExternalIP) && (len(cfg.AgentConfig.NodeExternalIPs) == 0) { + if cfg.Flannel.Backend != flannel.BackendNone { + if (cfg.Flannel.ExternalIP) && (len(cfg.AgentConfig.NodeExternalIPs) == 0) { logrus.Warnf("Server has flannel-external-ip flag set but this node does not set node-external-ip. Flannel will use internal address when connecting to this node.") - } else if (cfg.FlannelExternalIP) && (cfg.FlannelBackend != daemonconfig.FlannelBackendWireguardNative) { - logrus.Warnf("Flannel is using external addresses with an insecure backend: %v. Please consider using an encrypting flannel backend.", cfg.FlannelBackend) + } else if (cfg.Flannel.ExternalIP) && (cfg.Flannel.Backend != flannel.BackendWireguardNative) { + logrus.Warnf("Flannel is using external addresses with an insecure backend: %v. Please consider using an encrypting flannel backend.", cfg.Flannel.Backend) } if err := flannel.Prepare(ctx, cfg); err != nil { return err