mirror of
https://github.com/k3s-io/k3s.git
synced 2026-05-28 04:34:19 -04:00
Move embedded flannel and vpn config setup into embedded executor
Flannel and VPN setup shouldn't be done in generic agent config as it is only used with embeded executor's flannel CNI. Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
This commit is contained in:
Brad Davidson
Brad Davidson
parent
ec3cc04214
commit
efcf8eb7ac
3 changed files with 93 additions and 77 deletions
|
|
@ -16,7 +16,6 @@ import (
|
|||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
|
@ -32,7 +31,6 @@ import (
|
|||
"github.com/k3s-io/k3s/pkg/spegel"
|
||||
"github.com/k3s-io/k3s/pkg/util"
|
||||
"github.com/k3s-io/k3s/pkg/version"
|
||||
"github.com/k3s-io/k3s/pkg/vpn"
|
||||
pkgerrors "github.com/pkg/errors"
|
||||
certutil "github.com/rancher/dynamiclistener/cert"
|
||||
"github.com/rancher/wharfie/pkg/registries"
|
||||
|
|
@ -470,14 +468,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
|
|||
}
|
||||
apiServerURL := proxy.APIServerURL()
|
||||
|
||||
var flannelIface *net.Interface
|
||||
if controlConfig.FlannelBackend != config.FlannelBackendNone && len(envInfo.FlannelIface) > 0 {
|
||||
flannelIface, err = net.InterfaceByName(envInfo.FlannelIface)
|
||||
if err != nil {
|
||||
return nil, pkgerrors.WithMessagef(err, "unable to find interface %s", envInfo.FlannelIface)
|
||||
}
|
||||
}
|
||||
|
||||
clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
|
||||
if err := getHostFile(clientCAFile, info); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -504,47 +494,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
|
|||
newNodePasswordFile := filepath.Join(nodeConfigPath, "password")
|
||||
upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile)
|
||||
|
||||
// If there is a VPN, we must overwrite NodeIP and flannel interface
|
||||
var vpnInfo vpn.VPNInfo
|
||||
if envInfo.VPNAuth != "" {
|
||||
vpnInfo, err = vpn.GetVPNInfo(envInfo.VPNAuth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Pass ipv4, ipv6 or both depending on nodeIPs mode
|
||||
var vpnIPs []net.IP
|
||||
if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
|
||||
if vpnInfo.IPv6Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
|
||||
}
|
||||
} else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
|
||||
if vpnInfo.IPv4Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address)
|
||||
}
|
||||
|
||||
// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
|
||||
if len(vpnIPs) != 0 {
|
||||
logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs)
|
||||
if len(envInfo.NodeIP.Value()) != 0 {
|
||||
logrus.Warn("VPN provider overrides configured node-ip parameter")
|
||||
}
|
||||
if len(envInfo.NodeExternalIP.Value()) != 0 {
|
||||
logrus.Warn("VPN provider overrides node-external-ip parameter")
|
||||
}
|
||||
nodeIPs = vpnIPs
|
||||
flannelIface, err = net.InterfaceByName(vpnInfo.VPNInterface)
|
||||
if err != nil {
|
||||
return nil, pkgerrors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if controlConfig.ClusterIPRange != nil {
|
||||
if utilsnet.IPFamilyOfCIDR(controlConfig.ClusterIPRange) != utilsnet.IPFamilyOf(nodeIPs[0]) && len(nodeIPs) > 1 {
|
||||
firstNodeIP := nodeIPs[0]
|
||||
|
|
@ -639,7 +588,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
|
|||
SupervisorMetrics: controlConfig.SupervisorMetrics,
|
||||
Token: info.String(),
|
||||
}
|
||||
nodeConfig.FlannelIface = flannelIface
|
||||
nodeConfig.Images = filepath.Join(envInfo.DataDir, "agent", "images")
|
||||
nodeConfig.AgentConfig.NodeName = nodeName
|
||||
nodeConfig.AgentConfig.NodeConfigPath = nodeConfigPath
|
||||
|
|
@ -705,29 +653,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
|
|||
}
|
||||
nodeConfig.AgentConfig.NodeInternalDNSs = nodeInternalDNSs
|
||||
|
||||
nodeConfig.NoFlannel = nodeConfig.FlannelBackend == config.FlannelBackendNone
|
||||
if !nodeConfig.NoFlannel {
|
||||
hostLocal, err := exec.LookPath("host-local")
|
||||
if err != nil {
|
||||
return nil, pkgerrors.WithMessagef(err, "failed to find host-local")
|
||||
}
|
||||
|
||||
if envInfo.FlannelConf == "" {
|
||||
nodeConfig.FlannelConfFile = filepath.Join(envInfo.DataDir, "agent", "etc", "flannel", "net-conf.json")
|
||||
} else {
|
||||
nodeConfig.FlannelConfFile = envInfo.FlannelConf
|
||||
nodeConfig.FlannelConfOverride = true
|
||||
}
|
||||
nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)
|
||||
nodeConfig.AgentConfig.CNIConfDir = filepath.Join(envInfo.DataDir, "agent", "etc", "cni", "net.d")
|
||||
nodeConfig.AgentConfig.FlannelCniConfFile = envInfo.FlannelCniConfFile
|
||||
|
||||
// It does not make sense to use VPN without its flannel backend
|
||||
if envInfo.VPNAuth != "" {
|
||||
nodeConfig.FlannelBackend = vpnInfo.ProviderName
|
||||
}
|
||||
}
|
||||
|
||||
if nodeConfig.ImageServiceEndpoint != "" {
|
||||
nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ type Node struct {
|
|||
Docker bool
|
||||
ContainerRuntimeEndpoint string
|
||||
ImageServiceEndpoint string
|
||||
NoFlannel bool
|
||||
SELinux bool
|
||||
EnablePProf bool
|
||||
SupervisorMetrics bool
|
||||
|
|
|
|||
|
|
@ -7,7 +7,11 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
|
@ -19,10 +23,12 @@ import (
|
|||
"github.com/k3s-io/k3s/pkg/agent/flannel"
|
||||
"github.com/k3s-io/k3s/pkg/agent/netpol"
|
||||
"github.com/k3s-io/k3s/pkg/cli/cmds"
|
||||
"github.com/k3s-io/k3s/pkg/daemons/config"
|
||||
daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
|
||||
"github.com/k3s-io/k3s/pkg/signals"
|
||||
"github.com/k3s-io/k3s/pkg/util"
|
||||
"github.com/k3s-io/k3s/pkg/version"
|
||||
"github.com/k3s-io/k3s/pkg/vpn"
|
||||
pkgerrors "github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"k8s.io/apiserver/pkg/authentication/authenticator"
|
||||
|
|
@ -38,6 +44,7 @@ import (
|
|||
proxy "k8s.io/kubernetes/cmd/kube-proxy/app"
|
||||
sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
|
||||
kubelet "k8s.io/kubernetes/cmd/kubelet/app"
|
||||
utilsnet "k8s.io/utils/net"
|
||||
|
||||
// registering k3s cloud provider
|
||||
_ "github.com/k3s-io/k3s/pkg/cloudprovider"
|
||||
|
|
@ -74,6 +81,82 @@ func (e *Embedded) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node,
|
|||
}
|
||||
})
|
||||
|
||||
if nodeConfig.FlannelBackend != config.FlannelBackendNone {
|
||||
var err error
|
||||
|
||||
var flannelIface *net.Interface
|
||||
if len(cfg.FlannelIface) > 0 {
|
||||
flannelIface, err = net.InterfaceByName(cfg.FlannelIface)
|
||||
if err != nil {
|
||||
return pkgerrors.WithMessagef(err, "unable to find interface %s", cfg.FlannelIface)
|
||||
}
|
||||
}
|
||||
nodeConfig.FlannelIface = flannelIface
|
||||
|
||||
// If there is a VPN, we must overwrite NodeIP and flannel interface
|
||||
var vpnInfo vpn.VPNInfo
|
||||
if cfg.VPNAuth != "" {
|
||||
vpnInfo, err = vpn.GetVPNInfo(cfg.VPNAuth)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Pass ipv4, ipv6 or both depending on nodeIPs mode
|
||||
nodeIPs := nodeConfig.AgentConfig.NodeIPs
|
||||
var vpnIPs []net.IP
|
||||
if utilsnet.IsIPv4(nodeIPs[0]) && vpnInfo.IPv4Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
|
||||
if vpnInfo.IPv6Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
|
||||
}
|
||||
} else if utilsnet.IsIPv6(nodeIPs[0]) && vpnInfo.IPv6Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv6Address)
|
||||
if vpnInfo.IPv4Address != nil {
|
||||
vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address)
|
||||
}
|
||||
|
||||
// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
|
||||
if len(vpnIPs) != 0 {
|
||||
logrus.Infof("Node-ip changed to %v due to VPN", vpnIPs)
|
||||
if len(cfg.NodeIP.Value()) != 0 {
|
||||
logrus.Warn("VPN provider overrides configured node-ip parameter")
|
||||
}
|
||||
if len(cfg.NodeExternalIP.Value()) != 0 {
|
||||
logrus.Warn("VPN provider overrides node-external-ip parameter")
|
||||
}
|
||||
nodeIPs = vpnIPs
|
||||
flannelIface, err = net.InterfaceByName(vpnInfo.VPNInterface)
|
||||
if err != nil {
|
||||
return pkgerrors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set paths for embedded flannel if enabled
|
||||
hostLocal, err := exec.LookPath("host-local")
|
||||
if err != nil {
|
||||
return pkgerrors.WithMessagef(err, "failed to find host-local")
|
||||
}
|
||||
|
||||
if cfg.FlannelConf == "" {
|
||||
nodeConfig.FlannelConfFile = filepath.Join(cfg.DataDir, "agent", "etc", "flannel", "net-conf.json")
|
||||
} else {
|
||||
nodeConfig.FlannelConfFile = cfg.FlannelConf
|
||||
nodeConfig.FlannelConfOverride = true
|
||||
}
|
||||
nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)
|
||||
nodeConfig.AgentConfig.CNIConfDir = filepath.Join(cfg.DataDir, "agent", "etc", "cni", "net.d")
|
||||
nodeConfig.AgentConfig.FlannelCniConfFile = cfg.FlannelCniConfFile
|
||||
|
||||
// It does not make sense to use VPN without its flannel backend
|
||||
if cfg.VPNAuth != "" {
|
||||
nodeConfig.FlannelBackend = vpnInfo.ProviderName
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -254,7 +337,16 @@ func (e *Embedded) CRI(ctx context.Context, cfg *daemonconfig.Node) error {
|
|||
}
|
||||
|
||||
func (e *Embedded) CNI(ctx context.Context, wg *sync.WaitGroup, cfg *daemonconfig.Node) error {
|
||||
if !cfg.NoFlannel {
|
||||
if cfg.FlannelBackend != daemonconfig.FlannelBackendNone {
|
||||
if (cfg.FlannelExternalIP) && (len(cfg.AgentConfig.NodeExternalIPs) == 0) {
|
||||
logrus.Warnf("Server has flannel-external-ip flag set but this node does not set node-external-ip. Flannel will use internal address when connecting to this node.")
|
||||
} else if (cfg.FlannelExternalIP) && (cfg.FlannelBackend != daemonconfig.FlannelBackendWireguardNative) {
|
||||
logrus.Warnf("Flannel is using external addresses with an insecure backend: %v. Please consider using an encrypting flannel backend.", cfg.FlannelBackend)
|
||||
}
|
||||
if err := flannel.Prepare(ctx, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := flannel.Run(ctx, wg, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue