From 843cd606303649f54ab5695c3b03bc7064f17b49 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 17 Apr 2026 14:14:01 +0400 Subject: [PATCH 1/2] Upstream: proxy_next_upstream_delay directive The directive sets a timeout to wait before connecting to the next upstream. --- src/http/modules/ngx_http_proxy_module.c | 11 +++++++ src/http/ngx_http_upstream.c | 41 ++++++++++++++++++++++++ src/http/ngx_http_upstream.h | 3 ++ 3 files changed, 55 insertions(+) diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index e33dc37fd..8310d0211 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -575,6 +575,13 @@ static ngx_command_t ngx_http_proxy_commands[] = { offsetof(ngx_http_proxy_loc_conf_t, upstream.next_upstream_timeout), NULL }, + { ngx_string("proxy_next_upstream_delay"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.next_upstream_delay), + NULL }, + { ngx_string("proxy_pass_header"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_array_slot, @@ -3531,6 +3538,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->upstream.send_timeout = NGX_CONF_UNSET_MSEC; conf->upstream.read_timeout = NGX_CONF_UNSET_MSEC; conf->upstream.next_upstream_timeout = NGX_CONF_UNSET_MSEC; + conf->upstream.next_upstream_delay = NGX_CONF_UNSET_MSEC; conf->upstream.send_lowat = NGX_CONF_UNSET_SIZE; conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE; @@ -3673,6 +3681,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_msec_value(conf->upstream.next_upstream_timeout, prev->upstream.next_upstream_timeout, 0); + ngx_conf_merge_msec_value(conf->upstream.next_upstream_delay, + prev->upstream.next_upstream_delay, 0); + ngx_conf_merge_size_value(conf->upstream.send_lowat, prev->upstream.send_lowat, 0); diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index b64561369..243205802 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -99,6 +99,7 @@ static void ngx_http_upstream_dummy_handler(ngx_http_request_t *r, static void ngx_http_upstream_next(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_uint_t ft_type); static void ngx_http_upstream_cleanup(void *data); +static void ngx_http_upstream_connect_handler(ngx_event_t *ev); static void ngx_http_upstream_finalize_request(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_int_t rc); @@ -4725,6 +4726,42 @@ ngx_http_upstream_next(ngx_http_request_t *r, ngx_http_upstream_t *u, u->peer.connection = NULL; } + if (u->conf->next_upstream_delay) { + + if (u->event == NULL) { + u->event = ngx_pcalloc(r->pool, sizeof(ngx_event_t)); + if (u->event == NULL) { + ngx_http_upstream_finalize_request(r, u, + NGX_HTTP_INTERNAL_SERVER_ERROR); + return; + } + + u->event->handler = ngx_http_upstream_connect_handler; + u->event->data = r; + u->event->log = r->connection->log; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "next http upstream delay %M", + u->conf->next_upstream_delay); + + ngx_add_timer(u->event, u->conf->next_upstream_delay); + return; + } + + ngx_http_upstream_connect(r, u); +} + + +static void +ngx_http_upstream_connect_handler(ngx_event_t *ev) +{ + ngx_http_request_t *r; + ngx_http_upstream_t *u; + + r = ev->data; + u = r->upstream; + ngx_http_upstream_connect(r, u); } @@ -4822,6 +4859,10 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r, u->peer.connection = NULL; + if (u->event && u->event->timer_set) { + ngx_del_timer(u->event); + } + if (u->pipe) { u->pipe->upstream = NULL; } diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 0d48db788..c2eca2b6b 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -166,6 +166,7 @@ typedef struct { ngx_msec_t send_timeout; ngx_msec_t read_timeout; ngx_msec_t next_upstream_timeout; + ngx_msec_t next_upstream_delay; size_t send_lowat; size_t buffer_size; @@ -339,6 +340,8 @@ struct ngx_http_upstream_s { ngx_peer_connection_t peer; + ngx_event_t *event; + ngx_event_pipe_t *pipe; ngx_chain_t *request_bufs; From 73cd2e5bda65434ef0d99e2fb7f539b05fe824b0 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Sat, 18 Apr 2026 10:53:58 +0400 Subject: [PATCH 2/2] Upstream: proxy_next_upstream_unique directive Previously, nginx never picked the same peer twice during a proxy session. The proxy_next_upstream_unique directive allows to change this behavior and choose a peer from scratch each time. By default, the feature is enabled, which is the old behavior. --- src/event/ngx_event_connect.h | 1 + src/http/modules/ngx_http_proxy_module.c | 11 +++++++++++ src/http/modules/ngx_http_upstream_hash_module.c | 8 ++++++-- .../modules/ngx_http_upstream_least_conn_module.c | 4 +++- src/http/modules/ngx_http_upstream_random_module.c | 8 ++++++-- src/http/ngx_http_upstream.c | 6 ++++-- src/http/ngx_http_upstream.h | 1 + src/http/ngx_http_upstream_round_robin.c | 14 +++++++++++--- 8 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/event/ngx_event_connect.h b/src/event/ngx_event_connect.h index e428e7376..ca523f28d 100644 --- a/src/event/ngx_event_connect.h +++ b/src/event/ngx_event_connect.h @@ -69,6 +69,7 @@ struct ngx_peer_connection_s { unsigned transparent:1; unsigned so_keepalive:1; unsigned down:1; + unsigned unique:1; /* ngx_connection_log_error_e */ unsigned log_error:2; diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 8310d0211..a07bce5d3 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -582,6 +582,13 @@ static ngx_command_t ngx_http_proxy_commands[] = { offsetof(ngx_http_proxy_loc_conf_t, upstream.next_upstream_delay), NULL }, + { ngx_string("proxy_next_upstream_unique"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.next_upstream_unique), + NULL }, + { ngx_string("proxy_pass_header"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_array_slot, @@ -3530,6 +3537,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->upstream.request_buffering = NGX_CONF_UNSET; conf->upstream.ignore_client_abort = NGX_CONF_UNSET; conf->upstream.force_ranges = NGX_CONF_UNSET; + conf->upstream.next_upstream_unique = NGX_CONF_UNSET; conf->upstream.local = NGX_CONF_UNSET_PTR; conf->upstream.socket_keepalive = NGX_CONF_UNSET; @@ -3663,6 +3671,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.force_ranges, prev->upstream.force_ranges, 0); + ngx_conf_merge_value(conf->upstream.next_upstream_unique, + prev->upstream.next_upstream_unique, 1); + ngx_conf_merge_ptr_value(conf->upstream.local, prev->upstream.local, NULL); diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c index 1835aee80..0d990ed34 100644 --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -308,7 +308,9 @@ found: ngx_http_upstream_rr_peer_unlock(hp->rrp.peers, peer); ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); - hp->rrp.tried[n] |= m; + if (pc->unique) { + hp->rrp.tried[n] |= m; + } return NGX_OK; } @@ -713,7 +715,9 @@ found: n = best_i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << best_i % (8 * sizeof(uintptr_t)); - hp->rrp.tried[n] |= m; + if (pc->unique) { + hp->rrp.tried[n] |= m; + } return NGX_OK; } diff --git a/src/http/modules/ngx_http_upstream_least_conn_module.c b/src/http/modules/ngx_http_upstream_least_conn_module.c index 066f457d1..8625ca1d1 100644 --- a/src/http/modules/ngx_http_upstream_least_conn_module.c +++ b/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -271,7 +271,9 @@ best_chosen: n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); - rrp->tried[n] |= m; + if (pc->unique) { + rrp->tried[n] |= m; + } ngx_http_upstream_rr_peers_unlock(peers); diff --git a/src/http/modules/ngx_http_upstream_random_module.c b/src/http/modules/ngx_http_upstream_random_module.c index 72885cde8..344ece333 100644 --- a/src/http/modules/ngx_http_upstream_random_module.c +++ b/src/http/modules/ngx_http_upstream_random_module.c @@ -327,7 +327,9 @@ found: ngx_http_upstream_rr_peer_unlock(peers, peer); ngx_http_upstream_rr_peers_unlock(peers); - rrp->tried[n] |= m; + if (pc->unique) { + rrp->tried[n] |= m; + } return NGX_OK; } @@ -463,7 +465,9 @@ found: ngx_http_upstream_rr_peers_unlock(peers); - rrp->tried[n] |= m; + if (pc->unique) { + rrp->tried[n] |= m; + } return NGX_OK; } diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 243205802..30075e434 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -839,9 +839,10 @@ found: } u->peer.start_time = ngx_current_msec; + u->peer.unique = u->conf->next_upstream_unique; if (u->conf->next_upstream_tries - && u->peer.tries > u->conf->next_upstream_tries) + && (!u->peer.unique || u->peer.tries > u->conf->next_upstream_tries)) { u->peer.tries = u->conf->next_upstream_tries; } @@ -1282,9 +1283,10 @@ ngx_http_upstream_resolve_handler(ngx_resolver_ctx_t *ctx) ur->ctx = NULL; u->peer.start_time = ngx_current_msec; + u->peer.unique = u->conf->next_upstream_unique; if (u->conf->next_upstream_tries - && u->peer.tries > u->conf->next_upstream_tries) + && (!u->peer.unique || u->peer.tries > u->conf->next_upstream_tries)) { u->peer.tries = u->conf->next_upstream_tries; } diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index c2eca2b6b..43843cdd6 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -186,6 +186,7 @@ typedef struct { ngx_uint_t next_upstream; ngx_uint_t store_access; ngx_uint_t next_upstream_tries; + ngx_flag_t next_upstream_unique; ngx_flag_t buffering; ngx_flag_t request_buffering; ngx_flag_t pass_request_headers; diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 412281767..5fce4e98f 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -713,7 +713,7 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_wlock(peers); #if (NGX_HTTP_UPSTREAM_ZONE) - if (peers->config && rrp->config != *peers->config) { + if (pc->unique && peers->config && rrp->config != *peers->config) { goto busy; } #endif @@ -919,7 +919,9 @@ best_chosen: n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); - rrp->tried[n] |= m; + if (pc->unique) { + rrp->tried[n] |= m; + } if (now - best->checked > best->fail_timeout) { best->checked = now; @@ -1037,7 +1039,13 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, ngx_http_upstream_rr_peers_unlock(rrp->peers); - pc->tries = 0; + if (pc->unique) { + pc->tries = 0; + + } else if (pc->tries) { + pc->tries--; + } + return; }