From 07256cf7ab1b08b757fc96a4d18d700040b1d2ea Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 1 Feb 2021 11:11:17 -0800 Subject: [PATCH] Add ServiceIPRange and ServiceNodePortRange to agent config Signed-off-by: Brad Davidson --- pkg/agent/config/config.go | 8 ++++++++ pkg/cli/cmds/server.go | 25 ++++++++++++++++--------- pkg/cli/server/server.go | 19 ++++++++++++------- pkg/daemons/config/types.go | 4 ++++ pkg/daemons/control/server.go | 1 + 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 8eb72697767..c29cdcd95fb 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -489,6 +489,14 @@ func get(envInfo *cmds.Agent, proxy proxy.Proxy) (*config.Node, error) { nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange } + if controlConfig.ServiceIPRange != nil { + nodeConfig.AgentConfig.ServiceCIDR = *controlConfig.ServiceIPRange + } + + if controlConfig.ServiceNodePortRange != nil { + nodeConfig.AgentConfig.ServiceNodePortRange = *controlConfig.ServiceNodePortRange + } + nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index a2df2724f30..3e4bca055bf 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -13,15 +13,16 @@ const ( ) type Server struct { - ClusterCIDR string - AgentToken string - AgentTokenFile string - Token string - TokenFile string - ClusterSecret string - ServiceCIDR string - ClusterDNS string - ClusterDomain string + ClusterCIDR string + AgentToken string + AgentTokenFile string + Token string + TokenFile string + ClusterSecret string + ServiceCIDR string + ServiceNodePortRange string + ClusterDNS string + ClusterDomain string // The port which kubectl clients can access k8s HTTPSPort int // The port which custom k3s API runs on @@ -125,6 +126,12 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.ServiceCIDR, Value: "10.43.0.0/16", }, + cli.StringFlag{ + Name: "service-node-port-range", + Usage: "(networking) Port range to reserve for services with NodePort visibility", + Destination: &ServerConfig.ServiceNodePortRange, + Value: "30000-32767", + }, cli.StringFlag{ Name: "cluster-dns", Usage: "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index cd0cdbfe26d..83c5bcdf1f2 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -3,7 +3,7 @@ package server import ( "context" "fmt" - net2 "net" + "net" "os" "path/filepath" "strings" @@ -22,7 +22,7 @@ import ( "github.com/rancher/wrangler/pkg/signals" "github.com/sirupsen/logrus" "github.com/urfave/cli" - "k8s.io/apimachinery/pkg/util/net" + utilnet "k8s.io/apimachinery/pkg/util/net" kubeapiserverflag "k8s.io/component-base/cli/flag" "k8s.io/kubernetes/pkg/controlplane" @@ -145,15 +145,20 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP) } - _, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR) + _, serverConfig.ControlConfig.ClusterIPRange, err = net.ParseCIDR(cfg.ClusterCIDR) if err != nil { return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ClusterCIDR, err) } - _, serverConfig.ControlConfig.ServiceIPRange, err = net2.ParseCIDR(cfg.ServiceCIDR) + _, serverConfig.ControlConfig.ServiceIPRange, err = net.ParseCIDR(cfg.ServiceCIDR) if err != nil { return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err) } + serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange) + if err != nil { + return errors.Wrapf(err, "Invalid port range %s: %v", cfg.ServiceNodePortRange, err) + } + _, apiServerServiceIP, err := controlplane.ServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange) if err != nil { return err @@ -163,11 +168,11 @@ func run(app *cli.Context, cfg *cmds.Server) error { // If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 if cfg.ClusterDNS == "" { - serverConfig.ControlConfig.ClusterDNS = make(net2.IP, 4) + serverConfig.ControlConfig.ClusterDNS = make(net.IP, 4) copy(serverConfig.ControlConfig.ClusterDNS, serverConfig.ControlConfig.ServiceIPRange.IP.To4()) serverConfig.ControlConfig.ClusterDNS[3] = 10 } else { - serverConfig.ControlConfig.ClusterDNS = net2.ParseIP(cfg.ClusterDNS) + serverConfig.ControlConfig.ClusterDNS = net.ParseIP(cfg.ClusterDNS) } if cfg.DefaultLocalStoragePath == "" { @@ -287,7 +292,7 @@ func run(app *cli.Context, cfg *cmds.Server) error { func knownIPs(ips []string) []string { ips = append(ips, "127.0.0.1") - ip, err := net.ChooseHostInterface() + ip, err := utilnet.ChooseHostInterface() if err == nil { ips = append(ips, ip.String()) } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 2f05f3129da..edad438d86f 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -11,6 +11,7 @@ import ( "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/wrangler-api/pkg/generated/controllers/core" + utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/authentication/authenticator" ) @@ -56,6 +57,8 @@ type Agent struct { NodeConfigPath string ServingKubeletCert string ServingKubeletKey string + ServiceCIDR net.IPNet + ServiceNodePortRange utilnet.PortRange ClusterCIDR net.IPNet ClusterDNS net.IP ClusterDomain string @@ -102,6 +105,7 @@ type Control struct { Token string `json:"-"` ClusterIPRange *net.IPNet ServiceIPRange *net.IPNet + ServiceNodePortRange *utilnet.PortRange ClusterDNS net.IP ClusterDomain string NoCoreDNS bool diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 921637d0ffd..86cc1c61bae 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -190,6 +190,7 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") argsMap["service-account-signing-key-file"] = runtime.ServiceKey argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String() + argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String() argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort) if cfg.AdvertiseIP != "" { argsMap["advertise-address"] = cfg.AdvertiseIP