From 0aedf85236a5923a2d9c148c62b63f36bd58e0f2 Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Wed, 25 Feb 2026 12:06:05 +0100 Subject: [PATCH] Add custom-http-errors and default-backend annotations --- .../configuration-options.md | 1 + .../kubernetes/kubernetes-ingress-nginx.md | 14 +- .../kubernetes/ingress-nginx.md | 29 +-- pkg/config/dynamic/middlewares.go | 4 + pkg/config/dynamic/zz_generated.deepcopy.go | 21 ++ pkg/middlewares/customerrors/custom_errors.go | 40 ++- .../kubernetes/ingress-nginx/annotations.go | 3 + ...custom-http-errors-and-default-backend.yml | 23 ++ .../ingress-with-custom-http-errors.yml | 22 ++ ...ngress-with-default-backend-annotation.yml | 22 ++ .../ingress-nginx/fixtures/services.yml | 86 +++++++ .../kubernetes/ingress-nginx/kubernetes.go | 156 ++++++++--- .../ingress-nginx/kubernetes_test.go | 243 +++++++++++++++++- 13 files changed, 601 insertions(+), 63 deletions(-) create mode 100644 pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors-and-default-backend.yml create mode 100644 pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors.yml create mode 100644 pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-default-backend-annotation.yml diff --git a/docs/content/reference/install-configuration/configuration-options.md b/docs/content/reference/install-configuration/configuration-options.md index 4f3074c20..50eeb8ffd 100644 --- a/docs/content/reference/install-configuration/configuration-options.md +++ b/docs/content/reference/install-configuration/configuration-options.md @@ -395,6 +395,7 @@ THIS FILE MUST NOT BE EDITED BY HAND | providers.kubernetesingressnginx.certauthfilepath | Kubernetes certificate authority file path (not needed for in-cluster client). | | | providers.kubernetesingressnginx.clientbodybuffersize | Default buffer size for reading client request body. | 16384 | | providers.kubernetesingressnginx.controllerclass | Ingress Class Controller value this controller satisfies. | k8s.io/ingress-nginx | +| providers.kubernetesingressnginx.customhttperrors | Defines which status should result in calling the default backend to return an error page. | | | providers.kubernetesingressnginx.defaultbackendservice | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form 'namespace/name'. | | | providers.kubernetesingressnginx.disablesvcexternalname | Disable support for Services of type ExternalName. | false | | providers.kubernetesingressnginx.endpoint | Kubernetes server endpoint (required for external cluster client). | | diff --git a/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md b/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md index 902bdd561..1251d1e11 100644 --- a/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md +++ b/docs/content/reference/install-configuration/providers/kubernetes/kubernetes-ingress-nginx.md @@ -65,7 +65,10 @@ providers: proxyBuffering: false proxyBodySize: "1048576" # 1m proxyBufferSize: "8192" # 8k - proxyBuffersNumber: 8 + proxyBuffersNumber: 4 + customHTTPErrors: + - "404" + - "503" ``` ```toml tab="File (TOML)" @@ -86,7 +89,8 @@ providers: proxyBuffering = false proxyBodySize = "1048576" # 1m proxyBufferSize = "8192" # 8k - proxyBuffersNumber = 8 + proxyBuffersNumber = 4 + customHTTPErrors = ["404", "503"] ``` ```bash tab="CLI" @@ -102,7 +106,8 @@ providers: --providers.kubernetesingressnginx.proxybuffering=false --providers.kubernetesingressnginx.proxybodysize=1048576 # 1m --providers.kubernetesingressnginx.proxybuffersize=8192 # 8k ---providers.kubernetesingressnginx.proxybuffersnumber=8 +--providers.kubernetesingressnginx.proxybuffersnumber=4 +--providers.kubernetesingressnginx.customhttperrors=404,503 ``` ```yaml tab="Helm Chart Values" @@ -158,7 +163,8 @@ This provider watches for incoming Ingress events and automatically translates N | `providers.`
`kubernetesIngressNGINX.`
`proxybuffering` | Defines whether response buffering is enabled by default for all ingresses. | false | No | | `providers.`
`kubernetesIngressNGINX.`
`proxyBodySize` | Default maximum size of a client request body in bytes. | 1048576 | No | | `providers.`
`kubernetesIngressNGINX.`
`proxyBufferSize` | Default buffer size for reading the response body in bytes. | 8192 | No | -| `providers.`
`kubernetesIngressNGINX.`
`proxyBuffersNumber` | Default number of buffers for reading a response. | 8 | No | +| `providers.`
`kubernetesIngressNGINX.`
`proxyBuffersNumber` | Default number of buffers for reading a response. | 4 | No | +| `providers.`
`kubernetesIngressNGINX.`
`customHTTPErrors`
 | Defines which status should result in calling the default backend to return an error page. | [] | No | diff --git a/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md b/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md index 0af088d02..b47bdf7ce 100644 --- a/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md +++ b/docs/content/reference/routing-configuration/kubernetes/ingress-nginx.md @@ -321,13 +321,14 @@ The following annotations are organized by category for easier navigation. ### Load Balancing & Backend -| Annotation | Limitations / Notes | -|-------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| Annotation | Limitations / Notes | +|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| | `nginx.ingress.kubernetes.io/load-balance` | Only round_robin supported; ewma and IP hash not supported. | | `nginx.ingress.kubernetes.io/backend-protocol` | FCGI and AUTO_HTTP not supported. | | `nginx.ingress.kubernetes.io/service-upstream` | | | `nginx.ingress.kubernetes.io/upstream-vhost` | | | `nginx.ingress.kubernetes.io/custom-headers` | Header whitelisting, similar to `global-allowed-response-headers` NGINX config is not supported. | +| `nginx.ingress.kubernetes.io/default-backend` | Specifies a fallback service within the same namespace as the Ingress resource used to handle requests when the primary backend service has no active endpoints. If the specified service exposes multiple ports, the first port will receive the traffic. | ### CORS @@ -343,16 +344,17 @@ The following annotations are organized by category for easier navigation. ### Routing -| Annotation | Limitations / Notes | -|-------------------------------------------------------|--------------------------------------------------------------------------------------------| -| `nginx.ingress.kubernetes.io/app-root` | | -| `nginx.ingress.kubernetes.io/from-to-www-redirect` | Doesn't support wildcard hosts. | -| `nginx.ingress.kubernetes.io/use-regex` | | -| `nginx.ingress.kubernetes.io/rewrite-target` | | -| `nginx.ingress.kubernetes.io/permanent-redirect` | Defaults to a 301 Moved Permanently status code. | -| `nginx.ingress.kubernetes.io/permanent-redirect-code` | Only valid 3XX HTTP Status Codes are accepted. | -| `nginx.ingress.kubernetes.io/temporal-redirect` | Takes precedence over the `permanent-redirect` annotation. Defaults to a 302 Found status code. | -| `nginx.ingress.kubernetes.io/temporal-redirect-code` | Only valid 3XX HTTP Status Codes are accepted. | +| Annotation | Limitations / Notes | +|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `nginx.ingress.kubernetes.io/app-root` | | +| `nginx.ingress.kubernetes.io/from-to-www-redirect` | Doesn't support wildcard hosts. | +| `nginx.ingress.kubernetes.io/use-regex` | | +| `nginx.ingress.kubernetes.io/rewrite-target` | | +| `nginx.ingress.kubernetes.io/permanent-redirect` | Defaults to a 301 Moved Permanently status code. | +| `nginx.ingress.kubernetes.io/permanent-redirect-code` | Only valid 3XX HTTP Status Codes are accepted. | +| `nginx.ingress.kubernetes.io/temporal-redirect` | Takes precedence over the `permanent-redirect` annotation. Defaults to a 302 Found status code. | +| `nginx.ingress.kubernetes.io/temporal-redirect-code` | Only valid 3XX HTTP Status Codes are accepted. | +| `nginx.ingress.kubernetes.io/custom-http-errors` | Specifies a comma-separated list of HTTP status codes that should be intercepted and served by an error page backend. When any of these status codes occur, the request is forwarded to the global default backend, or to the backend defined by the [default-backend](#opt-nginx-ingress-kubernetes-iodefault-backend) annotation if specified. | ### IP Whitelist @@ -388,7 +390,6 @@ The following annotations are organized by category for easier navigation. - **Authentication**: Forward auth behaves differently and session caching is not supported. NGINX supports sub-request based auth, while Traefik forwards the original request. - **Session Affinity**: Only persistent mode is supported. - **Leader Election**: Not supported; no cluster mode with leader election. -- **Default Backend**: Only defaultBackend in Ingress spec is supported; the annotation is ignored. - **Load Balancing**: Only round_robin is supported; EWMA and IP hash are not supported. - **CORS**: NGINX responds with all configured headers unconditionally; Traefik handles headers differently between pre-flight and regular requests. - **TLS/Backend Protocols**: AUTO_HTTP, FCGI and some TLS options are not supported in Traefik. @@ -427,9 +428,7 @@ The following annotations are organized by category for easier navigation. | `nginx.ingress.kubernetes.io/canary-weight` | | | `nginx.ingress.kubernetes.io/canary-weight-total` | | | `nginx.ingress.kubernetes.io/configuration-snippet` | | -| `nginx.ingress.kubernetes.io/custom-http-errors` | | | `nginx.ingress.kubernetes.io/disable-proxy-intercept-errors` | | -| `nginx.ingress.kubernetes.io/default-backend` | Use `defaultBackend` in Ingress spec. | | `nginx.ingress.kubernetes.io/limit-rate-after` | | | `nginx.ingress.kubernetes.io/limit-rate` | | | `nginx.ingress.kubernetes.io/limit-whitelist` | | diff --git a/pkg/config/dynamic/middlewares.go b/pkg/config/dynamic/middlewares.go index 64340459f..434c654c6 100644 --- a/pkg/config/dynamic/middlewares.go +++ b/pkg/config/dynamic/middlewares.go @@ -268,6 +268,10 @@ type ErrorPage struct { // The {originalStatus} variable can be used in order to insert the upstream status code in the URL. // The {url} variable can be used in order to insert the escaped request URL. Query string `json:"query,omitempty" toml:"query,omitempty" yaml:"query,omitempty" export:"true"` + + // NginxHeaders defines the headers to forward to the Error page service. + // NginxHeaders option is unexposed to other providers than the IngressNGINX one. + NginxHeaders *http.Header `json:"nginxHeaders,omitempty" toml:"-" yaml:"-" label:"-" file:"-" kv:"-" export:"true"` } // +k8s:deepcopy-gen=true diff --git a/pkg/config/dynamic/zz_generated.deepcopy.go b/pkg/config/dynamic/zz_generated.deepcopy.go index faa7f70b8..5f342da8d 100644 --- a/pkg/config/dynamic/zz_generated.deepcopy.go +++ b/pkg/config/dynamic/zz_generated.deepcopy.go @@ -30,6 +30,8 @@ THE SOFTWARE. package dynamic import ( + http "net/http" + paersertypes "github.com/traefik/paerser/types" tls "github.com/traefik/traefik/v3/pkg/tls" types "github.com/traefik/traefik/v3/pkg/types" @@ -358,6 +360,25 @@ func (in *ErrorPage) DeepCopyInto(out *ErrorPage) { (*out)[key] = val } } + if in.NginxHeaders != nil { + in, out := &in.NginxHeaders, &out.NginxHeaders + *out = new(http.Header) + if **in != nil { + in, out := *in, *out + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + } return } diff --git a/pkg/middlewares/customerrors/custom_errors.go b/pkg/middlewares/customerrors/custom_errors.go index 193ad43ec..f0550edb3 100644 --- a/pkg/middlewares/customerrors/custom_errors.go +++ b/pkg/middlewares/customerrors/custom_errors.go @@ -16,6 +16,7 @@ import ( "github.com/traefik/traefik/v3/pkg/middlewares/observability" "github.com/traefik/traefik/v3/pkg/types" "github.com/vulcand/oxy/v2/utils" + "k8s.io/utils/ptr" ) // Compile time validation that the response recorder implements http interfaces correctly. @@ -32,12 +33,13 @@ type serviceBuilder interface { // customErrors is a middleware that provides the custom error pages. type customErrors struct { - name string - next http.Handler - backendHandler http.Handler - httpCodeRanges types.HTTPCodeRanges - backendQuery string - statusRewrites []statusRewrite + name string + next http.Handler + backendHandler http.Handler + httpCodeRanges types.HTTPCodeRanges + backendQuery string + statusRewrites []statusRewrite + forwardNginxHeaders http.Header } type statusRewrite struct { @@ -74,12 +76,13 @@ func New(ctx context.Context, next http.Handler, config dynamic.ErrorPage, servi } return &customErrors{ - name: name, - next: next, - backendHandler: backend, - httpCodeRanges: httpCodeRanges, - backendQuery: config.Query, - statusRewrites: statusRewrites, + name: name, + next: next, + backendHandler: backend, + httpCodeRanges: httpCodeRanges, + backendQuery: config.Query, + statusRewrites: statusRewrites, + forwardNginxHeaders: ptr.Deref(config.NginxHeaders, nil), }, nil } @@ -145,7 +148,18 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) { return } - utils.CopyHeaders(pageReq.Header, req.Header) + if len(c.forwardNginxHeaders) > 0 { + utils.CopyHeaders(pageReq.Header, c.forwardNginxHeaders) + pageReq.Header.Set("X-Code", strconv.Itoa(code)) + pageReq.Header.Set("X-Format", req.Header.Get("Accept")) + pageReq.Header.Set("X-Original-Uri", req.URL.RequestURI()) + if requestID := req.Header.Get("X-Request-ID"); requestID != "" { + pageReq.Header.Set("X-Request-ID", requestID) + } + } else { + utils.CopyHeaders(pageReq.Header, req.Header) + } + c.backendHandler.ServeHTTP(newCodeModifier(rw, code), pageReq.WithContext(req.Context())) } diff --git a/pkg/provider/kubernetes/ingress-nginx/annotations.go b/pkg/provider/kubernetes/ingress-nginx/annotations.go index b940a69f8..2a31a295f 100644 --- a/pkg/provider/kubernetes/ingress-nginx/annotations.go +++ b/pkg/provider/kubernetes/ingress-nginx/annotations.go @@ -73,6 +73,9 @@ type ingressConfig struct { CustomHeaders *string `annotation:"nginx.ingress.kubernetes.io/custom-headers"` UpstreamVhost *string `annotation:"nginx.ingress.kubernetes.io/upstream-vhost"` + CustomHTTPErrors *[]string `annotation:"nginx.ingress.kubernetes.io/custom-http-errors"` + DefaultBackend *string `annotation:"nginx.ingress.kubernetes.io/default-backend"` + // ProxyRequestBuffering controls whether request buffering is enabled. ProxyRequestBuffering *string `annotation:"nginx.ingress.kubernetes.io/proxy-request-buffering"` // ClientBodyBufferSize sets the size of the buffer used for reading request body. diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors-and-default-backend.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors-and-default-backend.yml new file mode 100644 index 000000000..5278ffa3d --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors-and-default-backend.yml @@ -0,0 +1,23 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-custom-http-errors-and-default-backend + namespace: default + annotations: + nginx.ingress.kubernetes.io/default-backend: whoami_b + nginx.ingress.kubernetes.io/custom-http-errors: "404,415" + +spec: + ingressClassName: nginx + rules: + - host: whoami.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 \ No newline at end of file diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors.yml new file mode 100644 index 000000000..5407c1d38 --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-custom-http-errors.yml @@ -0,0 +1,22 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-custom-http-errors + namespace: default + annotations: + nginx.ingress.kubernetes.io/custom-http-errors: "404,415" + +spec: + ingressClassName: nginx + rules: + - host: whoami.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: whoami + port: + number: 80 \ No newline at end of file diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-default-backend-annotation.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-default-backend-annotation.yml new file mode 100644 index 000000000..ceca9ad1a --- /dev/null +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/ingresses/ingress-with-default-backend-annotation.yml @@ -0,0 +1,22 @@ +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-with-default-backend-annotation + namespace: default + annotations: + nginx.ingress.kubernetes.io/default-backend: whoami_b + +spec: + ingressClassName: nginx + rules: + - host: whoami.localhost + http: + paths: + - path: / + pathType: Exact + backend: + service: + name: empty + port: + number: 80 \ No newline at end of file diff --git a/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml b/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml index b658083aa..f04d232d4 100644 --- a/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml +++ b/pkg/provider/kubernetes/ingress-nginx/fixtures/services.yml @@ -78,3 +78,89 @@ endpoints: - 10.10.0.6 conditions: ready: true + +--- +kind: Service +apiVersion: v1 +metadata: + name: whoami_b + namespace: default + +spec: + clusterIP: 10.10.10.2 + ports: + - name: web2 + protocol: TCP + port: 8000 + targetPort: web2 + - name: web + protocol: TCP + port: 80 + targetPort: web + selector: + app: whoami_b + task: whoami_b + +--- +kind: EndpointSlice +apiVersion: discovery.k8s.io/v1 +metadata: + name: whoami_b + namespace: default + labels: + kubernetes.io/service-name: whoami_b + +addressType: IPv4 +ports: + - name: web + port: 80 + - name: web2 + port: 8000 +endpoints: + - addresses: + - 10.10.0.7 + - 10.10.0.8 + conditions: + ready: true +--- +--- +kind: Service +apiVersion: v1 +metadata: + name: empty + namespace: default + +spec: + clusterIP: 10.10.10.3 + ports: + - name: web2 + protocol: TCP + port: 8000 + targetPort: web2 + - name: web + protocol: TCP + port: 80 + targetPort: web + selector: + app: empty + task: empty + +--- +kind: EndpointSlice +apiVersion: discovery.k8s.io/v1 +metadata: + name: empty + namespace: default + labels: + kubernetes.io/service-name: empty + +addressType: IPv4 +ports: + - name: web + port: 80 + - name: web2 + port: 8000 +endpoints: + - addresses: [] + conditions: + ready: true diff --git a/pkg/provider/kubernetes/ingress-nginx/kubernetes.go b/pkg/provider/kubernetes/ingress-nginx/kubernetes.go index 59e708936..f64bbd076 100644 --- a/pkg/provider/kubernetes/ingress-nginx/kubernetes.go +++ b/pkg/provider/kubernetes/ingress-nginx/kubernetes.go @@ -97,17 +97,17 @@ type Provider struct { DefaultBackendService string `description:"Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form 'namespace/name'." json:"defaultBackendService,omitempty" toml:"defaultBackendService,omitempty" yaml:"defaultBackendService,omitempty" export:"true"` DisableSvcExternalName bool `description:"Disable support for Services of type ExternalName." json:"disableSvcExternalName,omitempty" toml:"disableSvcExternalName,omitempty" yaml:"disableSvcExternalName,omitempty" export:"true"` - ProxyConnectTimeout int `description:"Amount of time to wait until a connection to a server can be established. Timeout value is unitless and in seconds." json:"proxyConnectTimeout,omitempty" toml:"proxyConnectTimeout,omitempty" yaml:"proxyConnectTimeout,omitempty" export:"true"` - ProxyReadTimeout int `description:"Amount of time between two successive read operations. Timeout value is unitless and in seconds." json:"proxyReadTimeout,omitempty" toml:"proxyReadTimeout,omitempty" yaml:"proxyReadTimeout,omitempty" export:"true"` - ProxySendTimeout int `description:"Amount of time between two successive write operations. Timeout value is unitless and in seconds." json:"proxySendTimeout,omitempty" toml:"proxySendTimeout,omitempty" yaml:"proxySendTimeout,omitempty" export:"true"` - // Configuration options available within the NGINX Ingress Controller ConfigMap. - ProxyRequestBuffering bool `description:"Defines whether to enable request buffering." json:"proxyRequestBuffering,omitempty" toml:"proxyRequestBuffering,omitempty" yaml:"proxyRequestBuffering,omitempty" export:"true"` - ClientBodyBufferSize int64 `description:"Default buffer size for reading client request body." json:"clientBodyBufferSize,omitempty" toml:"clientBodyBufferSize,omitempty" yaml:"clientBodyBufferSize,omitempty" export:"true"` - ProxyBodySize int64 `description:"Default maximum size of a client request body in bytes." json:"proxyBodySize,omitempty" toml:"proxyBodySize,omitempty" yaml:"proxyBodySize,omitempty" export:"true"` - ProxyBuffering bool `description:"Defines whether to enable response buffering." json:"proxyBuffering,omitempty" toml:"proxyBuffering,omitempty" yaml:"proxyBuffering,omitempty" export:"true"` - ProxyBufferSize int64 `description:"Default buffer size for reading the response body." json:"proxyBufferSize,omitempty" toml:"proxyBufferSize,omitempty" yaml:"proxyBufferSize,omitempty" export:"true"` - ProxyBuffersNumber int `description:"Default number of buffers for reading a response." json:"proxyBuffersNumber,omitempty" toml:"proxyBuffersNumber,omitempty" yaml:"proxyBuffersNumber,omitempty" export:"true"` + ProxyRequestBuffering bool `description:"Defines whether to enable request buffering." json:"proxyRequestBuffering,omitempty" toml:"proxyRequestBuffering,omitempty" yaml:"proxyRequestBuffering,omitempty" export:"true"` + ClientBodyBufferSize int64 `description:"Default buffer size for reading client request body." json:"clientBodyBufferSize,omitempty" toml:"clientBodyBufferSize,omitempty" yaml:"clientBodyBufferSize,omitempty" export:"true"` + ProxyBodySize int64 `description:"Default maximum size of a client request body in bytes." json:"proxyBodySize,omitempty" toml:"proxyBodySize,omitempty" yaml:"proxyBodySize,omitempty" export:"true"` + ProxyBuffering bool `description:"Defines whether to enable response buffering." json:"proxyBuffering,omitempty" toml:"proxyBuffering,omitempty" yaml:"proxyBuffering,omitempty" export:"true"` + ProxyBufferSize int64 `description:"Default buffer size for reading the response body." json:"proxyBufferSize,omitempty" toml:"proxyBufferSize,omitempty" yaml:"proxyBufferSize,omitempty" export:"true"` + ProxyBuffersNumber int `description:"Default number of buffers for reading a response." json:"proxyBuffersNumber,omitempty" toml:"proxyBuffersNumber,omitempty" yaml:"proxyBuffersNumber,omitempty" export:"true"` + ProxyConnectTimeout int `description:"Amount of time to wait until a connection to a server can be established. Timeout value is unitless and in seconds." json:"proxyConnectTimeout,omitempty" toml:"proxyConnectTimeout,omitempty" yaml:"proxyConnectTimeout,omitempty" export:"true"` + ProxyReadTimeout int `description:"Amount of time between two successive read operations. Timeout value is unitless and in seconds." json:"proxyReadTimeout,omitempty" toml:"proxyReadTimeout,omitempty" yaml:"proxyReadTimeout,omitempty" export:"true"` + ProxySendTimeout int `description:"Amount of time between two successive write operations. Timeout value is unitless and in seconds." json:"proxySendTimeout,omitempty" toml:"proxySendTimeout,omitempty" yaml:"proxySendTimeout,omitempty" export:"true"` + CustomHTTPErrors []string `description:"Defines which status should result in calling the default backend to return an error page." json:"customHTTPErrors,omitempty" toml:"customHTTPErrors,omitempty" yaml:"customHTTPErrors,omitempty" export:"true"` // NonTLSEntryPoints contains the names of entrypoints that are configured without TLS. NonTLSEntryPoints []string `json:"-" toml:"-" yaml:"-" label:"-" file:"-"` @@ -380,7 +380,7 @@ func (p *Provider) loadConfiguration(ctx context.Context) *dynamic.Configuration Service: defaultBackendName, } - if err := p.applyMiddlewares(ingress.Namespace, defaultBackendName, "", "", hosts, ingressConfig, hasTLS, rt, conf); err != nil { + if err := p.applyMiddlewares(ingress.Namespace, ingress.Name, defaultBackendName, "", "", ingress.Spec.DefaultBackend, hosts, ingressConfig, hasTLS, rt, conf); err != nil { logger.Error().Err(err).Msg("Error applying middlewares") } @@ -398,7 +398,7 @@ func (p *Provider) loadConfiguration(ctx context.Context) *dynamic.Configuration rtTLS.TLS.Options = clientAuthTLSOptionName } - if err := p.applyMiddlewares(ingress.Namespace, defaultBackendTLSName, "", "", hosts, ingressConfig, false, rtTLS, conf); err != nil { + if err := p.applyMiddlewares(ingress.Namespace, ingress.Name, defaultBackendTLSName, "", "", ingress.Spec.DefaultBackend, hosts, ingressConfig, false, rtTLS, conf); err != nil { logger.Error().Err(err).Msg("Error applying middlewares") } @@ -475,7 +475,7 @@ func (p *Provider) loadConfiguration(ctx context.Context) *dynamic.Configuration Service: key, } - if err := p.applyMiddlewares(ingress.Namespace, key, "", "", hosts, ingressConfig, hasTLS, rt, conf); err != nil { + if err := p.applyMiddlewares(ingress.Namespace, ingress.Name, key, "", "", ingress.Spec.DefaultBackend, hosts, ingressConfig, hasTLS, rt, conf); err != nil { logger.Error().Err(err).Msg("Error applying middlewares") } @@ -492,7 +492,7 @@ func (p *Provider) loadConfiguration(ctx context.Context) *dynamic.Configuration rtTLS.TLS.Options = clientAuthTLSOptionName } - if err := p.applyMiddlewares(ingress.Namespace, key+"-tls", "", "", hosts, ingressConfig, false, rtTLS, conf); err != nil { + if err := p.applyMiddlewares(ingress.Namespace, ingress.Name, key+"-tls", "", "", ingress.Spec.DefaultBackend, hosts, ingressConfig, false, rtTLS, conf); err != nil { logger.Error().Err(err).Msg("Error applying middlewares") } @@ -561,7 +561,7 @@ func (p *Provider) loadConfiguration(ctx context.Context) *dynamic.Configuration conf.HTTP.ServersTransports[namedServersTransport.Name] = namedServersTransport.ServersTransport } - if err := p.applyMiddlewares(ingress.Namespace, routerKey, pa.Path, rule.Host, hosts, ingressConfig, hasTLS, rt, conf); err != nil { + if err := p.applyMiddlewares(ingress.Namespace, ingress.Name, routerKey, pa.Path, rule.Host, &pa.Backend, hosts, ingressConfig, hasTLS, rt, conf); err != nil { logger.Error().Err(err).Msg("Error applying middlewares") } } @@ -676,6 +676,18 @@ func (p *Provider) buildPassthroughService(namespace string, backend netv1.Ingre return &dynamic.TCPService{LoadBalancer: lb}, nil } +func getPort(service *corev1.Service, backend netv1.IngressBackend) (string, corev1.ServicePort, bool) { + for _, p := range service.Spec.Ports { + // A port with number 0 or an empty name is not allowed, this case is there for the default backend service. + if (backend.Service.Port.Number == 0 && backend.Service.Port.Name == "") || + (backend.Service.Port.Number == p.Port || (backend.Service.Port.Name == p.Name && len(p.Name) > 0)) { + return p.Name, p, true + } + } + + return "", corev1.ServicePort{}, false +} + func (p *Provider) getBackendAddresses(namespace string, backend netv1.IngressBackend, cfg ingressConfig) ([]backendAddress, error) { service, err := p.k8sClient.GetService(namespace, backend.Service.Name) if err != nil { @@ -686,19 +698,7 @@ func (p *Provider) getBackendAddresses(namespace string, backend netv1.IngressBa return nil, errors.New("externalName services not allowed") } - var portName string - var portSpec corev1.ServicePort - var match bool - for _, p := range service.Spec.Ports { - // A port with number 0 or an empty name is not allowed, this case is there for the default backend service. - if (backend.Service.Port.Number == 0 && backend.Service.Port.Name == "") || - (backend.Service.Port.Number == p.Port || (backend.Service.Port.Name == p.Name && len(p.Name) > 0)) { - portName = p.Name - portSpec = p - match = true - break - } - } + portName, portSpec, match := getPort(service, backend) if !match { return nil, errors.New("service port not found") } @@ -712,7 +712,40 @@ func (p *Provider) getBackendAddresses(namespace string, backend netv1.IngressBa return []backendAddress{{Address: net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(portSpec.Port)))}}, nil } - endpointSlices, err := p.k8sClient.GetEndpointSlicesForService(namespace, backend.Service.Name) + addresses, err := p.getBackendAddressesFromEndpointSlices(namespace, backend.Service.Name, portName) + if err != nil { + return nil, fmt.Errorf("getting backend addresses: %w", err) + } + + defaultBackend := ptr.Deref(cfg.DefaultBackend, "") + if defaultBackend == "" || defaultBackend == backend.Service.Name || len(addresses) > 0 { + return addresses, nil + } + + serviceDefaultBackend, err := p.k8sClient.GetService(namespace, defaultBackend) + if err != nil { + return nil, fmt.Errorf("getting service: %w", err) + } + + if p.DisableSvcExternalName && serviceDefaultBackend.Spec.Type == corev1.ServiceTypeExternalName { + return nil, errors.New("externalName services not allowed") + } + + portName, _, match = getPort(serviceDefaultBackend, netv1.IngressBackend{Service: &netv1.IngressServiceBackend{Name: defaultBackend}}) + if !match { + return nil, errors.New("service port not found") + } + + // If the default backend has no endpoints, + // and if there is no default-backend-service configured, + // the fallback with Ingress NGINX is to serve a 404, + // but here, we will later build an empty server load-balancer which serves a 503. + // TODO: make the built service return a 404. + return p.getBackendAddressesFromEndpointSlices(namespace, defaultBackend, portName) +} + +func (p *Provider) getBackendAddressesFromEndpointSlices(namespace, name, portName string) ([]backendAddress, error) { + endpointSlices, err := p.k8sClient.GetEndpointSlicesForService(namespace, name) if err != nil { return nil, fmt.Errorf("getting endpointslices: %w", err) } @@ -877,7 +910,12 @@ func (p *Provider) loadCertificates(ctx context.Context, ingress *netv1.Ingress, return nil } -func (p *Provider) applyMiddlewares(namespace, routerKey, rulePath, ruleHost string, hosts map[string]bool, ingressConfig ingressConfig, hasTLS bool, rt *dynamic.Router, conf *dynamic.Configuration) error { +func (p *Provider) applyMiddlewares(namespace, ingressName, routerKey, rulePath, ruleHost string, backend *netv1.IngressBackend, hosts map[string]bool, ingressConfig ingressConfig, hasTLS bool, rt *dynamic.Router, conf *dynamic.Configuration) error { + err := p.applyCustomHTTPErrors(namespace, ingressName, routerKey, backend, ingressConfig, rt, conf) + if err != nil { + return err + } + applyAppRootConfiguration(routerKey, ingressConfig, rt, conf) applyFromToWwwRedirect(hosts, ruleHost, routerKey, ingressConfig, rt, conf) applyRedirect(routerKey, ingressConfig, rt, conf) @@ -913,7 +951,65 @@ func (p *Provider) applyMiddlewares(namespace, routerKey, rulePath, ruleHost str if err := p.applyCustomHeaders(routerKey, ingressConfig, rt, conf); err != nil { return fmt.Errorf("applying custom headers: %w", err) } + return nil +} +func (p *Provider) applyCustomHTTPErrors(namespace, ingressName, routerName string, targetedService *netv1.IngressBackend, config ingressConfig, rt *dynamic.Router, conf *dynamic.Configuration) error { + customHTTPErrors := ptr.Deref(config.CustomHTTPErrors, p.CustomHTTPErrors) + if len(customHTTPErrors) == 0 { + return nil + } + + if targetedService == nil { + return errors.New("targeted ingress backend is nil") + } + + if targetedService.Service == nil { + return errors.New("targeted ingress backend has no service") + } + + // TODO: here we always use the default backend as a fallback, but it is not guaranteed to be created, + // so we should check if it exists before and create a dummy service if not, which is too complicated to check without pre computed model. + serviceName := defaultBackendName + if defaultBackend := ptr.Deref(config.DefaultBackend, ""); defaultBackend != "" { + backend := netv1.IngressBackend{Service: &netv1.IngressServiceBackend{Name: defaultBackend}} + service, err := p.buildService(namespace, backend, config) + if err != nil { + return err + } + + serviceName = fmt.Sprintf("default-backend-%s", routerName) + conf.HTTP.Services[serviceName] = service + } + + k8sServiceName := targetedService.Service.Name + serviceK8s, err := p.k8sClient.GetService(namespace, k8sServiceName) + if err != nil { + return fmt.Errorf("getting service: %w", err) + } + + _, portSpec, ok := getPort(serviceK8s, *targetedService) + if !ok { + return fmt.Errorf("port not found for service %s", k8sServiceName) + } + + customErrorMiddlewareName := routerName + "-custom-http-errors" + headers := http.Header(map[string][]string{ + "X-Namespaces": {namespace}, + "X-Ingress-Name": {ingressName}, + "X-Service-Name": {k8sServiceName}, + "X-Service-Port": {strconv.Itoa(int(portSpec.Port))}, + }) + + conf.HTTP.Middlewares[customErrorMiddlewareName] = &dynamic.Middleware{ + Errors: &dynamic.ErrorPage{ + Status: customHTTPErrors, + Service: serviceName, + NginxHeaders: &headers, + }, + } + + rt.Middlewares = append(rt.Middlewares, customErrorMiddlewareName) return nil } diff --git a/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go b/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go index 9c7ac57f1..199a14c59 100644 --- a/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go +++ b/pkg/provider/kubernetes/ingress-nginx/kubernetes_test.go @@ -2601,6 +2601,245 @@ func TestLoadIngresses(t *testing.T) { }, }, }, + { + desc: "Custom HTTP Errors and Default backend annotation", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/ingress-with-custom-http-errors-and-default-backend.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-custom-http-errors-and-default-backend-rule-0-path-0": { + Rule: "Host(`whoami.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-ingress-with-custom-http-errors-and-default-backend-whoami-80", + Middlewares: []string{"default-ingress-with-custom-http-errors-and-default-backend-rule-0-path-0-custom-http-errors"}, + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-ingress-with-custom-http-errors-and-default-backend-rule-0-path-0-custom-http-errors": { + Errors: &dynamic.ErrorPage{ + Status: []string{"404", "415"}, + Service: "default-backend-default-ingress-with-custom-http-errors-and-default-backend-rule-0-path-0", + NginxHeaders: &http.Header{ + "X-Namespaces": {"default"}, + "X-Ingress-Name": {"ingress-with-custom-http-errors-and-default-backend"}, + "X-Service-Name": {"whoami"}, + "X-Service-Port": {"80"}, + }, + }, + }, + }, + Services: map[string]*dynamic.Service{ + "default-ingress-with-custom-http-errors-and-default-backend-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + ServersTransport: "default-ingress-with-custom-http-errors-and-default-backend", + }, + }, + "default-backend-default-ingress-with-custom-http-errors-and-default-backend-rule-0-path-0": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.7:8000", + }, + { + URL: "http://10.10.0.8:8000", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-ingress-with-custom-http-errors-and-default-backend": { + ForwardingTimeouts: &dynamic.ForwardingTimeouts{ + DialTimeout: ptypes.Duration(60 * time.Second), + ReadTimeout: ptypes.Duration(60 * time.Second), + WriteTimeout: ptypes.Duration(60 * time.Second), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Custom HTTP Errors", + defaultBackendServiceName: "whoami_b", + defaultBackendServiceNamespace: "default", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/ingress-with-custom-http-errors.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-custom-http-errors-rule-0-path-0": { + Rule: "Host(`whoami.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-ingress-with-custom-http-errors-whoami-80", + Middlewares: []string{"default-ingress-with-custom-http-errors-rule-0-path-0-custom-http-errors"}, + }, + "default-backend": { + Rule: "PathPrefix(`/`)", + RuleSyntax: "default", + Priority: math.MinInt32, + Service: "default-backend", + }, + "default-backend-tls": { + Rule: "PathPrefix(`/`)", + RuleSyntax: "default", + Priority: math.MinInt32, + TLS: &dynamic.RouterTLSConfig{}, + Service: "default-backend", + }, + }, + Middlewares: map[string]*dynamic.Middleware{ + "default-ingress-with-custom-http-errors-rule-0-path-0-custom-http-errors": { + Errors: &dynamic.ErrorPage{ + Status: []string{"404", "415"}, + Service: "default-backend", + NginxHeaders: &http.Header{ + "X-Namespaces": {"default"}, + "X-Ingress-Name": {"ingress-with-custom-http-errors"}, + "X-Service-Name": {"whoami"}, + "X-Service-Port": {"80"}, + }, + }, + }, + }, + Services: map[string]*dynamic.Service{ + "default-ingress-with-custom-http-errors-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + ServersTransport: "default-ingress-with-custom-http-errors", + }, + }, + "default-backend": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.7:8000", + }, + { + URL: "http://10.10.0.8:8000", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-ingress-with-custom-http-errors": { + ForwardingTimeouts: &dynamic.ForwardingTimeouts{ + DialTimeout: ptypes.Duration(60 * time.Second), + ReadTimeout: ptypes.Duration(60 * time.Second), + WriteTimeout: ptypes.Duration(60 * time.Second), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Default backend annotation", + paths: []string{ + "services.yml", + "ingressclasses.yml", + "ingresses/ingress-with-default-backend-annotation.yml", + }, + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-ingress-with-default-backend-annotation-rule-0-path-0": { + Rule: "Host(`whoami.localhost`) && Path(`/`)", + RuleSyntax: "default", + Service: "default-ingress-with-default-backend-annotation-empty-80", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-ingress-with-default-backend-annotation-empty-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.7:8000", + }, + { + URL: "http://10.10.0.8:8000", + }, + }, + Strategy: "wrr", + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: dynamic.DefaultFlushInterval, + }, + ServersTransport: "default-ingress-with-default-backend-annotation", + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-ingress-with-default-backend-annotation": { + ForwardingTimeouts: &dynamic.ForwardingTimeouts{ + DialTimeout: ptypes.Duration(60 * time.Second), + ReadTimeout: ptypes.Duration(60 * time.Second), + WriteTimeout: ptypes.Duration(60 * time.Second), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, { desc: "Buffering with proxy body size of 10MB", paths: []string{ @@ -3122,7 +3361,9 @@ func TestLoadIngresses(t *testing.T) { ServersTransports: map[string]*dynamic.ServersTransport{ "default-ingress-with-auth-tls-pass-certificate-to-upstream": { ForwardingTimeouts: &dynamic.ForwardingTimeouts{ - DialTimeout: ptypes.Duration(60 * time.Second), + DialTimeout: ptypes.Duration(60 * time.Second), + ReadTimeout: ptypes.Duration(60 * time.Second), + WriteTimeout: ptypes.Duration(60 * time.Second), }, }, },