From dcbe752df54e178b4575786476e8843c3e2cd052 Mon Sep 17 00:00:00 2001 From: Kevin Pollet Date: Wed, 3 Jun 2026 11:06:05 +0200 Subject: [PATCH] Change default values and expose configuration for Kubernetes client QPS and Burst Co-authored-by: Anatole Lucet --- docs/content/migrate/v3.md | 14 ++++++++ .../configuration-options.md | 2 ++ .../kubernetes/kubernetes-gateway.md | 34 ++++++++++--------- pkg/provider/kubernetes/gateway/client.go | 17 ++++++---- pkg/provider/kubernetes/gateway/kubernetes.go | 13 +++++-- 5 files changed, 54 insertions(+), 26 deletions(-) diff --git a/docs/content/migrate/v3.md b/docs/content/migrate/v3.md index eb89268256..cb6b08cbb9 100644 --- a/docs/content/migrate/v3.md +++ b/docs/content/migrate/v3.md @@ -9,6 +9,20 @@ This guide provides detailed migration steps for upgrading between different Tra --- +## v3.6.19 + +### Kubernetes Gateway API Provider + +Starting with `v3.6.19`, the QPS and Burst values of the Kubernetes client used by the Kubernetes Gateway API provider have been increased to `50` and `100` respectively (10x the default values of the Kubernetes client). + +The Kubernetes Gateway API provider writes status updates intensively to comply with the Kubernetes Gateway API specification. +This change helps avoid performance issues related to Kubernetes API rate limiting, which can increase the setup time when a new routing configuration is built. + +These values are configurable through the [`kubernetesGateway.qps`](../reference/install-configuration/providers/kubernetes/kubernetes-gateway.md#opt-providers-kubernetesgateway-qps) +and [`kubernetesGateway.burst`](../reference/install-configuration/providers/kubernetes/kubernetes-gateway.md#opt-providers-kubernetesgateway-burst) provider options. + +--- + ## v3.6.18 ### BasicAuth Middleware diff --git a/docs/content/reference/install-configuration/configuration-options.md b/docs/content/reference/install-configuration/configuration-options.md index 50732d098a..807a110d86 100644 --- a/docs/content/reference/install-configuration/configuration-options.md +++ b/docs/content/reference/install-configuration/configuration-options.md @@ -359,6 +359,7 @@ THIS FILE MUST NOT BE EDITED BY HAND | providers.kubernetescrd.throttleduration | Ingress refresh throttle duration | 0 | | providers.kubernetescrd.token | Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token. | | | providers.kubernetesgateway | Enables Kubernetes Gateway API provider. | false | +| providers.kubernetesgateway.burst | Defines the maximum burst of requests to the Kubernetes API server. | 100 | | providers.kubernetesgateway.certauthfilepath | Kubernetes certificate authority file path (not needed for in-cluster client). | | | providers.kubernetesgateway.crossprovidernamespaces | List of namespaces from which Gateway API routes are allowed to declare TraefikService backendRef references. | | | providers.kubernetesgateway.endpoint | Kubernetes server endpoint (required for external cluster client). | | @@ -366,6 +367,7 @@ THIS FILE MUST NOT BE EDITED BY HAND | providers.kubernetesgateway.labelselector | Kubernetes label selector to select specific GatewayClasses. | | | providers.kubernetesgateway.namespaces | Kubernetes namespaces. | | | providers.kubernetesgateway.nativelbbydefault | Defines whether to use Native Kubernetes load-balancing by default. | false | +| providers.kubernetesgateway.qps | Defines the maximum QPS to the Kubernetes API server. Setting this to a negative value will disable client-side ratelimiting. | 50 | | providers.kubernetesgateway.statusaddress.hostname | Hostname used for Kubernetes Gateway status address. | | | providers.kubernetesgateway.statusaddress.ip | IP used to set Kubernetes Gateway status address. | | | providers.kubernetesgateway.statusaddress.service | Published Kubernetes Service to copy status addresses from. | | diff --git a/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-gateway.md b/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-gateway.md index 95c8c32292..b0fdbe029c 100644 --- a/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-gateway.md +++ b/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-gateway.md @@ -67,22 +67,24 @@ providers: -| Field | Description | Default | Required | -|:----------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------|:---------| -| `providers.providersThrottleDuration` | Minimum amount of time to wait for, after a configuration reload, before taking into account any new configuration refresh event.
If multiple events occur within this time, only the most recent one is taken into account, and all others are discarded.
**This option cannot be set per provider, but the throttling algorithm applies to each of them independently.** | 2s | No | -| `providers.kubernetesGateway.endpoint` | Server endpoint URL.
More information [here](#endpoint). | "" | No | -| `providers.kubernetesGateway.experimentalChannel` | Toggles support for the Experimental Channel resources ([Gateway API release channels documentation](https://gateway-api.sigs.k8s.io/concepts/versioning/#release-channels)).
(ex: `TCPRoute` and `TLSRoute`) | false | No | -| `providers.kubernetesGateway.token` | Bearer token used for the Kubernetes client configuration. | "" | No | -| `providers.kubernetesGateway.certAuthFilePath` | Path to the certificate authority file.
Used for the Kubernetes client configuration. | "" | No | -| `providers.kubernetesGateway.namespaces` | Array of namespaces to watch.
If left empty, watch all namespaces. | [] | No | -| `providers.kubernetesGateway.labelselector` | Allow filtering on `GatewayClass` only. If left empty, Traefik processes all GatewayClass objects in the configured namespaces.
See [label-selectors](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) for details. | "" | No | -| `providers.kubernetesGateway.throttleDuration` | Minimum amount of time to wait between two Kubernetes events before producing a new configuration.
This prevents a Kubernetes cluster that updates many times per second from continuously changing your Traefik configuration.
If empty, every event is caught. | 0s | No | -| `providers.kubernetesGateway.nativeLBByDefault` | Defines whether to use Native Kubernetes load-balancing mode by default. For more information, please check out the `traefik.io/service.nativelb` service annotation documentation. | false | No | -| `providers.kubernetesGateway.`
`statusAddress.hostname` | Hostname copied to the Gateway `status.addresses`. | "" | No | -| `providers.kubernetesGateway.`
`statusAddress.ip` | IP address copied to the Gateway `status.addresses`, and currently only supports one IP value (IPv4 or IPv6). | "" | No | -| `providers.kubernetesGateway.`
`statusAddress.service.namespace` | The namespace of the Kubernetes service to copy status addresses from.
When using third parties tools like External-DNS, this option can be used to copy the service `loadbalancer.status` (containing the service's endpoints IPs) to the Gateway `status.addresses`. | "" | No | -| `providers.kubernetesGateway.`
`statusAddress.service.name` | The name of the Kubernetes service to copy status addresses from.
When using third parties tools like External-DNS, this option can be used to copy the service `loadbalancer.status` (containing the service's endpoints IPs) to the Gateway `status.addresses`. | "" | No | -| `providers.kubernetesGateway.crossProviderNamespaces` | List of namespaces from which Gateway API routes (`HTTPRoute`, `TCPRoute`, `TLSRoute`) are allowed to declare a `backendRef` of kind `TraefikService`.
When unset, all namespaces are allowed. When set to `[]`, every such backendRef is rejected and the route is dropped. | [] | No | +| Field | Description | Default | Required | +|:----------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------|:---------| +| `providers.providersThrottleDuration` | Minimum amount of time to wait for, after a configuration reload, before taking into account any new configuration refresh event.
If multiple events occur within this time, only the most recent one is taken into account, and all others are discarded.
**This option cannot be set per provider, but the throttling algorithm applies to each of them independently.** | 2s | No | +| `providers.kubernetesGateway.endpoint` | Server endpoint URL.
More information [here](#endpoint). | "" | No | +| `providers.kubernetesGateway.experimentalChannel` | Toggles support for the Experimental Channel resources ([Gateway API release channels documentation](https://gateway-api.sigs.k8s.io/concepts/versioning/#release-channels)).
(ex: `TCPRoute` and `TLSRoute`) | false | No | +| `providers.kubernetesGateway.token` | Bearer token used for the Kubernetes client configuration. | "" | No | +| `providers.kubernetesGateway.certAuthFilePath` | Path to the certificate authority file.
Used for the Kubernetes client configuration. | "" | No | +| `providers.kubernetesGateway.namespaces` | Array of namespaces to watch.
If left empty, watch all namespaces. | [] | No | +| `providers.kubernetesGateway.labelselector` | Allow filtering on `GatewayClass` only. If left empty, Traefik processes all GatewayClass objects in the configured namespaces.
See [label-selectors](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) for details. | "" | No | +| `providers.kubernetesGateway.throttleDuration` | Minimum amount of time to wait between two Kubernetes events before producing a new configuration.
This prevents a Kubernetes cluster that updates many times per second from continuously changing your Traefik configuration.
If empty, every event is caught. | 0s | No | +| `providers.kubernetesGateway.nativeLBByDefault` | Defines whether to use Native Kubernetes load-balancing mode by default. For more information, please check out the `traefik.io/service.nativelb` service annotation documentation. | false | No | +| `providers.kubernetesGateway.`
`statusAddress.hostname` | Hostname copied to the Gateway `status.addresses`. | "" | No | +| `providers.kubernetesGateway.`
`statusAddress.ip` | IP address copied to the Gateway `status.addresses`, and currently only supports one IP value (IPv4 or IPv6). | "" | No | +| `providers.kubernetesGateway.`
`statusAddress.service.namespace` | The namespace of the Kubernetes service to copy status addresses from.
When using third parties tools like External-DNS, this option can be used to copy the service `loadbalancer.status` (containing the service's endpoints IPs) to the Gateway `status.addresses`. | "" | No | +| `providers.kubernetesGateway.`
`statusAddress.service.name` | The name of the Kubernetes service to copy status addresses from.
When using third parties tools like External-DNS, this option can be used to copy the service `loadbalancer.status` (containing the service's endpoints IPs) to the Gateway `status.addresses`. | "" | No | +| `providers.kubernetesGateway.crossProviderNamespaces` | List of namespaces from which Gateway API routes (`HTTPRoute`, `TCPRoute`, `TLSRoute`) are allowed to declare a `backendRef` of kind `TraefikService`.
When unset, all namespaces are allowed. When set to `[]`, every such backendRef is rejected and the route is dropped. | [] | No | +| providers.kubernetesgateway.qps | Defines the maximum QPS to the Kubernetes API server. Setting this to a negative value will disable client-side ratelimiting. | 50 | No | +| providers.kubernetesgateway.burst | Defines the maximum burst of requests to the Kubernetes API server. | 100 | No | diff --git a/pkg/provider/kubernetes/gateway/client.go b/pkg/provider/kubernetes/gateway/client.go index fd383d89e4..37c9a85dad 100644 --- a/pkg/provider/kubernetes/gateway/client.go +++ b/pkg/provider/kubernetes/gateway/client.go @@ -49,7 +49,10 @@ type clientWrapper struct { experimentalChannel bool } -func createClientFromConfig(c *rest.Config) (*clientWrapper, error) { +func createClientFromConfig(c *rest.Config, qps, burst int) (*clientWrapper, error) { + c.QPS = float32(qps) + c.Burst = burst + csGateway, err := gateclientset.NewForConfig(c) if err != nil { return nil, err @@ -75,7 +78,7 @@ func newClientImpl(csKube kclientset.Interface, csGateway gateclientset.Interfac // newInClusterClient returns a new Provider client that is expected to run // inside the cluster. -func newInClusterClient(endpoint string) (*clientWrapper, error) { +func newInClusterClient(endpoint string, qps, burst int) (*clientWrapper, error) { config, err := rest.InClusterConfig() if err != nil { return nil, fmt.Errorf("failed to create in-cluster configuration: %w", err) @@ -85,20 +88,20 @@ func newInClusterClient(endpoint string) (*clientWrapper, error) { config.Host = endpoint } - return createClientFromConfig(config) + return createClientFromConfig(config, qps, burst) } -func newExternalClusterClientFromFile(file string) (*clientWrapper, error) { +func newExternalClusterClientFromFile(file string, qps, burst int) (*clientWrapper, error) { configFromFlags, err := clientcmd.BuildConfigFromFlags("", file) if err != nil { return nil, err } - return createClientFromConfig(configFromFlags) + return createClientFromConfig(configFromFlags, qps, burst) } // newExternalClusterClient returns a new Provider client that may run outside of the cluster. // The endpoint parameter must not be empty. -func newExternalClusterClient(endpoint, caFilePath string, token types.FileOrContent) (*clientWrapper, error) { +func newExternalClusterClient(endpoint, caFilePath string, token types.FileOrContent, qps, burst int) (*clientWrapper, error) { if endpoint == "" { return nil, errors.New("endpoint missing for external cluster client") } @@ -122,7 +125,7 @@ func newExternalClusterClient(endpoint, caFilePath string, token types.FileOrCon config.TLSClientConfig = rest.TLSClientConfig{CAData: caData} } - return createClientFromConfig(config) + return createClientFromConfig(config, qps, burst) } // WatchAll starts namespace-specific controllers for all relevant kinds. diff --git a/pkg/provider/kubernetes/gateway/kubernetes.go b/pkg/provider/kubernetes/gateway/kubernetes.go index 2883c42c13..ddc4903884 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes.go +++ b/pkg/provider/kubernetes/gateway/kubernetes.go @@ -64,6 +64,8 @@ const ( type Provider struct { Endpoint string `description:"Kubernetes server endpoint (required for external cluster client)." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` Token types.FileOrContent `description:"Kubernetes bearer token (not needed for in-cluster client). It accepts either a token value or a file path to the token." json:"token,omitempty" toml:"token,omitempty" yaml:"token,omitempty" loggable:"false"` + QPS int `description:"Defines the maximum QPS to the Kubernetes API server. Setting this to a negative value will disable client-side ratelimiting." json:"qps,omitempty" toml:"qps,omitempty" yaml:"qps,omitempty" export:"true"` + Burst int `description:"Defines the maximum burst of requests to the Kubernetes API server." json:"burst,omitempty" toml:"burst,omitempty" yaml:"burst,omitempty" export:"true"` CertAuthFilePath string `description:"Kubernetes certificate authority file path (not needed for in-cluster client)." json:"certAuthFilePath,omitempty" toml:"certAuthFilePath,omitempty" yaml:"certAuthFilePath,omitempty"` Namespaces []string `description:"Kubernetes namespaces." json:"namespaces,omitempty" toml:"namespaces,omitempty" yaml:"namespaces,omitempty" export:"true"` LabelSelector string `description:"Kubernetes label selector to select specific GatewayClasses." json:"labelSelector,omitempty" toml:"labelSelector,omitempty" yaml:"labelSelector,omitempty" export:"true"` @@ -85,6 +87,11 @@ type Provider struct { client *clientWrapper } +func (p *Provider) SetDefaults() { + p.QPS = 50 // the default value for the QPS is 10x the default Kubernetes client QPS value. + p.Burst = 100 // the default value for the Burst is 10x the default Kubernetes client Burst value. +} + // Entrypoint defines the available entry points. type Entrypoint struct { Address string @@ -274,13 +281,13 @@ func (p *Provider) newK8sClient(ctx context.Context) (*clientWrapper, error) { switch { case os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "": logger.Info().Str("endpoint", p.Endpoint).Msg("Creating in-cluster Provider client") - client, err = newInClusterClient(p.Endpoint) + client, err = newInClusterClient(p.Endpoint, p.QPS, p.Burst) case os.Getenv("KUBECONFIG") != "": logger.Info().Msgf("Creating cluster-external Provider client from KUBECONFIG %s", os.Getenv("KUBECONFIG")) - client, err = newExternalClusterClientFromFile(os.Getenv("KUBECONFIG")) + client, err = newExternalClusterClientFromFile(os.Getenv("KUBECONFIG"), p.QPS, p.Burst) default: logger.Info().Str("endpoint", p.Endpoint).Msg("Creating cluster-external Provider client") - client, err = newExternalClusterClient(p.Endpoint, p.CertAuthFilePath, p.Token) + client, err = newExternalClusterClient(p.Endpoint, p.CertAuthFilePath, p.Token, p.QPS, p.Burst) } if err != nil {