From 9986b28380de77c86e0c8cef6b64640ac5d0277b Mon Sep 17 00:00:00 2001 From: Erdem Agaoglu Date: Thu, 1 Dec 2016 16:29:45 +0300 Subject: [PATCH 1/4] Set read-timeout for http.Server This also specifies a timeout for idle client connections, which may cause "too many open files" errors. See #2238 --- cmd/prometheus/config.go | 4 ++++ web/web.go | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index e8302a03ae..234e15f1aa 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -80,6 +80,10 @@ func init() { &cfg.web.ListenAddress, "web.listen-address", ":9090", "Address to listen on for the web interface, API, and telemetry.", ) + cfg.fs.DurationVar( + &cfg.web.ReadTimeout, "web.read-timeout", 30*time.Second, + "Maximum duration before timing out read of the request, and closing idle connections.", + ) cfg.fs.StringVar( &cfg.prometheusURL, "web.external-url", "", "The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically.", diff --git a/web/web.go b/web/web.go index 828839eb0b..ea571abac8 100644 --- a/web/web.go +++ b/web/web.go @@ -111,6 +111,7 @@ type Options struct { Flags map[string]string ListenAddress string + ReadTimeout time.Duration ExternalURL *url.URL RoutePrefix string MetricsPath string @@ -247,9 +248,10 @@ func (h *Handler) Reload() <-chan chan error { func (h *Handler) Run() { log.Infof("Listening on %s", h.options.ListenAddress) server := &http.Server{ - Addr: h.options.ListenAddress, - Handler: h.router, - ErrorLog: log.NewErrorLogger(), + Addr: h.options.ListenAddress, + Handler: h.router, + ErrorLog: log.NewErrorLogger(), + ReadTimeout: h.options.ReadTimeout, } h.listenErrCh <- server.ListenAndServe() } From e487477a17e90d1b145eb2fe1a67005715fee250 Mon Sep 17 00:00:00 2001 From: Erdem Agaoglu Date: Tue, 6 Dec 2016 12:45:59 +0300 Subject: [PATCH 2/4] LimitListener to limit max number of connections This also drops tcp keep-alive in ListenAndServe but it's no longer necessary since we now close idle connections long before that. --- cmd/prometheus/config.go | 4 ++++ web/web.go | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index 234e15f1aa..5410a6f623 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -84,6 +84,10 @@ func init() { &cfg.web.ReadTimeout, "web.read-timeout", 30*time.Second, "Maximum duration before timing out read of the request, and closing idle connections.", ) + cfg.fs.IntVar( + &cfg.web.MaxConnections, "web.max-connections", 100, + "Maximum number of simultaneous connections.", + ) cfg.fs.StringVar( &cfg.prometheusURL, "web.external-url", "", "The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived automatically.", diff --git a/web/web.go b/web/web.go index ea571abac8..e0320fb6a8 100644 --- a/web/web.go +++ b/web/web.go @@ -37,6 +37,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/route" "golang.org/x/net/context" + "golang.org/x/net/netutil" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/notifier" @@ -112,6 +113,7 @@ type Options struct { ListenAddress string ReadTimeout time.Duration + MaxConnections int ExternalURL *url.URL RoutePrefix string MetricsPath string @@ -253,7 +255,13 @@ func (h *Handler) Run() { ErrorLog: log.NewErrorLogger(), ReadTimeout: h.options.ReadTimeout, } - h.listenErrCh <- server.ListenAndServe() + listener, err := net.Listen("tcp", h.options.ListenAddress) + if err != nil { + h.listenErrCh <- err + } else { + limitedListener := netutil.LimitListener(listener, h.options.MaxConnections) + h.listenErrCh <- server.Serve(limitedListener) + } } func (h *Handler) alerts(w http.ResponseWriter, r *http.Request) { From 2260079c127b38e08a64f2aa037955d3bc7da092 Mon Sep 17 00:00:00 2001 From: Erdem Agaoglu Date: Tue, 6 Dec 2016 12:52:02 +0300 Subject: [PATCH 3/4] Vendor x/net/netutil --- vendor/golang.org/x/net/netutil/listen.go | 48 +++++++++++++++++++++++ vendor/vendor.json | 6 +++ 2 files changed, 54 insertions(+) create mode 100644 vendor/golang.org/x/net/netutil/listen.go diff --git a/vendor/golang.org/x/net/netutil/listen.go b/vendor/golang.org/x/net/netutil/listen.go new file mode 100644 index 0000000000..b317ba2e6a --- /dev/null +++ b/vendor/golang.org/x/net/netutil/listen.go @@ -0,0 +1,48 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package netutil provides network utility functions, complementing the more +// common ones in the net package. +package netutil // import "golang.org/x/net/netutil" + +import ( + "net" + "sync" +) + +// LimitListener returns a Listener that accepts at most n simultaneous +// connections from the provided Listener. +func LimitListener(l net.Listener, n int) net.Listener { + return &limitListener{l, make(chan struct{}, n)} +} + +type limitListener struct { + net.Listener + sem chan struct{} +} + +func (l *limitListener) acquire() { l.sem <- struct{}{} } +func (l *limitListener) release() { <-l.sem } + +func (l *limitListener) Accept() (net.Conn, error) { + l.acquire() + c, err := l.Listener.Accept() + if err != nil { + l.release() + return nil, err + } + return &limitListenerConn{Conn: c, release: l.release}, nil +} + +type limitListenerConn struct { + net.Conn + releaseOnce sync.Once + release func() +} + +func (l *limitListenerConn) Close() error { + err := l.Conn.Close() + l.releaseOnce.Do(l.release) + return err +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 4afb1fcb79..aae150ab6a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -695,6 +695,12 @@ "revision": "c589d0c9f0d81640c518354c7bcae77d99820aa3", "revisionTime": "2016-09-30T00:14:02Z" }, + { + "checksumSHA1": "7WASrg0PEueWDDRHkFhEEN6Qrms=", + "path": "golang.org/x/net/netutil", + "revision": "bc3663df0ac92f928d419e31e0d2af22e683a5a2", + "revisionTime": "2016-06-21T20:48:10Z" + }, { "checksumSHA1": "mktBVED98G2vv+OKcSgtnFVZC1Y=", "path": "golang.org/x/oauth2", From 054f8ebbfb420b47923f997c79de773877f2e66b Mon Sep 17 00:00:00 2001 From: Erdem Agaoglu Date: Tue, 6 Dec 2016 17:45:19 +0300 Subject: [PATCH 4/4] Increase default max-connections --- cmd/prometheus/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index 5410a6f623..af124cc26c 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -85,7 +85,7 @@ func init() { "Maximum duration before timing out read of the request, and closing idle connections.", ) cfg.fs.IntVar( - &cfg.web.MaxConnections, "web.max-connections", 100, + &cfg.web.MaxConnections, "web.max-connections", 512, "Maximum number of simultaneous connections.", ) cfg.fs.StringVar(