From 36d2998f362b14d12d566f865e6026e05dbf8b46 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 18 Jul 2024 19:39:39 +0300 Subject: [PATCH 1/2] Upstream: using the "Age" header when caching responses. As long as the "Age" header is present and not ignored, it is now respected when caching responses per "Cache-Control: max-age", reducing cache validity time. --- src/http/ngx_http_upstream.c | 70 ++++++++++++++++++++++++++++++++++-- src/http/ngx_http_upstream.h | 4 +++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index f177f0a28..72e5bf74d 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -139,6 +139,8 @@ static ngx_int_t ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_upstream_process_vary(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); +static ngx_int_t ngx_http_upstream_process_age(ngx_http_request_t *r, + ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t ngx_http_upstream_copy_header_line(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); static ngx_int_t @@ -300,6 +302,10 @@ static ngx_http_upstream_header_t ngx_http_upstream_headers_in[] = { ngx_http_upstream_copy_multi_header_lines, offsetof(ngx_http_headers_out_t, link), 0 }, + { ngx_string("Age"), + ngx_http_upstream_process_age, 0, + ngx_http_upstream_copy_header_line, 0, 0 }, + { ngx_string("X-Accel-Expires"), ngx_http_upstream_process_accel_expires, 0, ngx_http_upstream_copy_header_line, 0, 0 }, @@ -496,6 +502,7 @@ ngx_conf_bitmask_t ngx_http_upstream_ignore_headers_masks[] = { { ngx_string("Cache-Control"), NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL }, { ngx_string("Set-Cookie"), NGX_HTTP_UPSTREAM_IGN_SET_COOKIE }, { ngx_string("Vary"), NGX_HTTP_UPSTREAM_IGN_VARY }, + { ngx_string("Age"), NGX_HTTP_UPSTREAM_IGN_AGE }, { ngx_null_string, 0 } }; @@ -5118,14 +5125,16 @@ ngx_http_upstream_process_cache_control(ngx_http_request_t *r, return NGX_OK; } - if (n == 0) { + if (n <= u->headers_in.age_n) { u->headers_in.no_cache = 1; return NGX_OK; } - r->cache->valid_sec = ngx_min((ngx_uint_t) ngx_time() + n, + r->cache->valid_sec = ngx_min((ngx_uint_t) ngx_time() + + n - u->headers_in.age_n, NGX_MAX_INT_T_VALUE); u->headers_in.expired = 0; + u->headers_in.max_age = 1; } extensions: @@ -5306,6 +5315,7 @@ ngx_http_upstream_process_accel_expires(ngx_http_request_t *r, r->cache->valid_sec = ngx_time() + n; u->headers_in.no_cache = 0; u->headers_in.expired = 0; + u->headers_in.max_age = 0; return NGX_OK; } } @@ -5319,6 +5329,7 @@ ngx_http_upstream_process_accel_expires(ngx_http_request_t *r, r->cache->valid_sec = n; u->headers_in.no_cache = 0; u->headers_in.expired = 0; + u->headers_in.max_age = 0; } } #endif @@ -5569,6 +5580,61 @@ ngx_http_upstream_process_vary(ngx_http_request_t *r, } +static ngx_int_t +ngx_http_upstream_process_age(ngx_http_request_t *r, + ngx_table_elt_t *h, ngx_uint_t offset) +{ + ngx_http_upstream_t *u; + + u = r->upstream; + + if (u->headers_in.age) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent duplicate header line: \"%V: %V\", " + "previous value: \"%V: %V\", ignored", + &h->key, &h->value, + &u->headers_in.age->key, + &u->headers_in.age->value); + h->hash = 0; + return NGX_OK; + } + + u->headers_in.age = h; + h->next = NULL; + +#if (NGX_HTTP_CACHE) + { + ngx_int_t n; + + if (u->conf->ignore_headers & NGX_HTTP_UPSTREAM_IGN_AGE) { + return NGX_OK; + } + + if (r->cache == NULL || !u->cacheable) { + return NGX_OK; + } + + n = ngx_atoi(h->value.data, h->value.len); + + if (n != NGX_ERROR) { + u->headers_in.age_n = n; + + if (u->headers_in.max_age) { + if (n >= r->cache->valid_sec - ngx_time()) { + u->headers_in.no_cache = 1; + return NGX_OK; + } + + r->cache->valid_sec -= n; + } + } + } +#endif + + return NGX_OK; +} + + static ngx_int_t ngx_http_upstream_copy_header_line(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset) diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 0d48db788..f5a3e2b9a 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -55,6 +55,7 @@ #define NGX_HTTP_UPSTREAM_IGN_XA_BUFFERING 0x00000080 #define NGX_HTTP_UPSTREAM_IGN_XA_CHARSET 0x00000100 #define NGX_HTTP_UPSTREAM_IGN_VARY 0x00000200 +#define NGX_HTTP_UPSTREAM_IGN_AGE 0x00000400 #define NGX_HTTP_UPSTREAM_NOTIFY_HEADER 0x1 @@ -302,14 +303,17 @@ typedef struct { ngx_table_elt_t *cache_control; ngx_table_elt_t *set_cookie; + ngx_table_elt_t *age; off_t content_length_n; time_t last_modified_time; + time_t age_n; unsigned connection_close:1; unsigned chunked:1; unsigned no_cache:1; unsigned expired:1; + unsigned max_age:1; } ngx_http_upstream_headers_in_t; From 86081bed10cb20cea7e72643024bec52fb426260 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 18 Jul 2024 19:39:45 +0300 Subject: [PATCH 2/2] Upstream: $upstream_cache_age variable. The variable reflects response age, including the time spent in the cache and the upstream response age as obtained from the "Age" header. If the response wasn't cached, the variable reflects the "Age" header of the upstream response. If the intended use case is to cache responses as per HTTP/1.1 caching model, the $upstream_cache_age variable can be used to provide the "Age" header with the "add_header" directive, such as: add_header Age $upstream_cache_age; This now removes the "Age" header if it was present. Further, the "expires" directives now removes the "Age" header if it was present in the response, as the "expires" directive assumes zero age when it adds "Expires" and "Cache-Control" headers. --- .../modules/ngx_http_headers_filter_module.c | 9 +++ src/http/ngx_http_request.h | 1 + src/http/ngx_http_upstream.c | 63 ++++++++++++++++++- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/src/http/modules/ngx_http_headers_filter_module.c b/src/http/modules/ngx_http_headers_filter_module.c index f72044097..acfdb273f 100644 --- a/src/http/modules/ngx_http_headers_filter_module.c +++ b/src/http/modules/ngx_http_headers_filter_module.c @@ -100,6 +100,10 @@ static ngx_http_set_header_t ngx_http_set_headers[] = { offsetof(ngx_http_headers_out_t, etag), ngx_http_set_response_header }, + { ngx_string("Age"), + offsetof(ngx_http_headers_out_t, age), + ngx_http_set_response_header }, + { ngx_null_string, 0, NULL } }; @@ -383,6 +387,11 @@ ngx_http_set_expires(ngx_http_request_t *r, ngx_http_headers_conf_t *conf) } } + if (r->headers_out.age) { + r->headers_out.age->hash = 0; + r->headers_out.age = NULL; + } + e = r->headers_out.expires; if (e == NULL) { diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index 7a77498eb..89e680807 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -281,6 +281,7 @@ typedef struct { ngx_table_elt_t *cache_control; ngx_table_elt_t *link; + ngx_table_elt_t *age; ngx_str_t *override_charset; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 72e5bf74d..f30a45f75 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -27,6 +27,8 @@ static ngx_int_t ngx_http_upstream_cache_last_modified(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_upstream_cache_etag(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_upstream_cache_age(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); #endif static void ngx_http_upstream_init_request(ngx_http_request_t *r); @@ -304,7 +306,8 @@ static ngx_http_upstream_header_t ngx_http_upstream_headers_in[] = { { ngx_string("Age"), ngx_http_upstream_process_age, 0, - ngx_http_upstream_copy_header_line, 0, 0 }, + ngx_http_upstream_copy_header_line, + offsetof(ngx_http_headers_out_t, age), 0 }, { ngx_string("X-Accel-Expires"), ngx_http_upstream_process_accel_expires, 0, @@ -457,6 +460,10 @@ static ngx_http_variable_t ngx_http_upstream_vars[] = { ngx_http_upstream_cache_etag, 0, NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + { ngx_string("upstream_cache_age"), NULL, + ngx_http_upstream_cache_age, 0, + NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + #endif { ngx_string("upstream_http_"), NULL, ngx_http_upstream_header_variable, @@ -6366,6 +6373,60 @@ ngx_http_upstream_cache_etag(ngx_http_request_t *r, return NGX_OK; } + +static ngx_int_t +ngx_http_upstream_cache_age(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + time_t now, age; + + if (r->upstream == NULL) { + v->not_found = 1; + return NGX_OK; + } + + if (!r->cached + || r->cache == NULL + || r->upstream->cache_status == NGX_HTTP_CACHE_REVALIDATED) + { + if (r->upstream->headers_in.age == NULL) { + v->not_found = 1; + return NGX_OK; + } + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->len = r->upstream->headers_in.age->value.len; + v->data = r->upstream->headers_in.age->value.data; + + return NGX_OK; + } + + p = ngx_pnalloc(r->pool, NGX_TIME_T_LEN); + if (p == NULL) { + return NGX_ERROR; + } + + now = ngx_time(); + age = now - r->cache->date; + + if (r->cache->date > now) { + age = 0; + } + + age += r->upstream->headers_in.age_n; + + v->len = ngx_sprintf(p, "%T", age) - p; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = p; + + return NGX_OK; +} + #endif