From 0b8a175511fb7d3330b481aac8eb42e7590fdb14 Mon Sep 17 00:00:00 2001 From: Daniel Salzman Date: Fri, 2 Jul 2021 13:37:57 +0200 Subject: [PATCH] XDP-TCP: configuration cleanup --- doc/man/knot.conf.5in | 24 ++++++++++++++---------- doc/reference.rst | 32 ++++++++++++++++++-------------- src/knot/conf/base.c | 10 +++++++--- src/knot/conf/base.h | 5 +++-- src/knot/conf/schema.c | 4 ++-- src/knot/conf/schema.h | 12 ++++++------ src/knot/conf/tools.c | 10 ++++++---- src/knot/server/server.c | 12 ++++++++---- src/knot/server/xdp-handler.c | 14 +++++++------- tests/knot/test_confio.c | 11 ++++++----- 10 files changed, 77 insertions(+), 57 deletions(-) diff --git a/doc/man/knot.conf.5in b/doc/man/knot.conf.5in index 7e6991e88..b20bd3b72 100644 --- a/doc/man/knot.conf.5in +++ b/doc/man/knot.conf.5in @@ -432,9 +432,9 @@ Various options related to XDP listening, especially TCP. .ft C xdp: listen: STR[@INT] | ADDR[@INT] ... - xdp\-tcp: BOOL + tcp: BOOL tcp\-max\-clients: INT - tcp\-max\-inbufs: SIZE + tcp\-inbuf\-max\-size: SIZE tcp\-idle\-close: TIME tcp\-idle\-reset: TIME route\-check: BOOL @@ -472,31 +472,35 @@ working TCP. .UNINDENT .sp \fIDefault:\fP not set -.SS xdp\-tcp +.SS tcp .sp -Also answer TCP traffic (queries) with XDP workers. +Also process DNS over TCP traffic with XDP workers. +.sp +Change of this parameter requires restart of the Knot server to take effect. .sp \fBWARNING:\fP .INDENT 0.0 .INDENT 3.5 This feature is highly experimental and it may eat your hamster as well as any -other hamsters connected to the network. It is especially discouraged to serve -large zone transfers through XDP. +other hamsters connected to the network. Multi\-message zone transfer isn\(aqt +supported through XDP. .UNINDENT .UNINDENT +.sp +\fIDefault:\fP off .SS tcp\-max\-clients .sp A maximum number of TCP clients connected in parallel. .sp \fIDefault:\fP 1000000 (one million) -.SS tcp\-max\-inbufs +.SS tcp\-inbuf\-max\-size .sp -Maximum cumulative size of RAM used for buffers of incomplete +Maximum cumulative size of memory used for buffers of incompletely received messages. .sp -\fIMinimum:\fP 1 Megabyte +\fIMinimum:\fP 1 MiB .sp -\fIDefault:\fP 100 Megabytes +\fIDefault:\fP 100 MiB .SS tcp\-idle\-close .sp Time in seconds, after which any idle connection is gracefully closed. diff --git a/doc/reference.rst b/doc/reference.rst index feb1fc0a7..7da047d6e 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -454,9 +454,9 @@ Various options related to XDP listening, especially TCP. xdp: listen: STR[@INT] | ADDR[@INT] ... - xdp-tcp: BOOL + tcp: BOOL tcp-max-clients: INT - tcp-max-inbufs: SIZE + tcp-inbuf-max-size: SIZE tcp-idle-close: TIME tcp-idle-reset: TIME route-check: BOOL @@ -487,17 +487,21 @@ Change of this parameter requires restart of the Knot server to take effect. *Default:* not set -.. _xdp_xdp-tcp: +.. _xdp_tcp: -xdp-tcp -------- +tcp +--- -Also answer TCP traffic (queries) with XDP workers. +Also process DNS over TCP traffic with XDP workers. + +Change of this parameter requires restart of the Knot server to take effect. .. WARNING:: This feature is highly experimental and it may eat your hamster as well as any - other hamsters connected to the network. It is especially discouraged to serve - large zone transfers through XDP. + other hamsters connected to the network. Multi-message zone transfer isn't + supported through XDP. + +*Default:* off .. _xdp_tcp-max-clients: @@ -508,17 +512,17 @@ A maximum number of TCP clients connected in parallel. *Default:* 1000000 (one million) -.. _xdp_tcp-max-inbufs: +.. _xdp_tcp-inbuf-max-size: -tcp-max-inbufs --------------- +tcp-inbuf-max-size +------------------ -Maximum cumulative size of RAM used for buffers of incomplete +Maximum cumulative size of memory used for buffers of incompletely received messages. -*Minimum:* 1 Megabyte +*Minimum:* 1 MiB -*Default:* 100 Megabytes +*Default:* 100 MiB .. _xdp_tcp-idle-close: diff --git a/src/knot/conf/base.c b/src/knot/conf/base.c index 412a79ee0..921b1e0e3 100644 --- a/src/knot/conf/base.c +++ b/src/knot/conf/base.c @@ -125,6 +125,7 @@ static void init_cache( static bool first_init = true; static bool running_tcp_reuseport; static bool running_socket_affinity; + static bool running_xdp_tcp; static bool running_route_check; static size_t running_udp_threads; static size_t running_tcp_threads; @@ -134,6 +135,7 @@ static void init_cache( if (first_init || reinit_cache) { running_tcp_reuseport = conf_get_bool(conf, C_SRV, C_TCP_REUSEPORT); running_socket_affinity = conf_get_bool(conf, C_SRV, C_SOCKET_AFFINITY); + running_xdp_tcp = conf_get_bool(conf, C_XDP, C_TCP); running_route_check = conf_get_bool(conf, C_XDP, C_ROUTE_CHECK); running_udp_threads = conf_udp_threads(conf); running_tcp_threads = conf_tcp_threads(conf); @@ -182,10 +184,10 @@ static void init_cache( conf->cache.srv_tcp_max_clients = conf_tcp_max_clients(conf); val = conf_get(conf, C_XDP, C_TCP_MAX_CLIENTS); - conf->cache.xdp_tcp_max_conns = conf_int(&val); + conf->cache.xdp_tcp_max_clients = conf_int(&val); - val = conf_get(conf, C_XDP, C_TCP_MAX_INBUFS); - conf->cache.xdp_tcp_inbufs_size = conf_int(&val); + val = conf_get(conf, C_XDP, C_TCP_INBUF_MAX_SIZE); + conf->cache.xdp_tcp_inbuf_max_size = conf_int(&val); val = conf_get(conf, C_XDP, C_TCP_IDLE_CLOSE); conf->cache.xdp_tcp_idle_close = conf_int(&val); @@ -193,6 +195,8 @@ static void init_cache( val = conf_get(conf, C_XDP, C_TCP_IDLE_RESET); conf->cache.xdp_tcp_idle_reset = conf_int(&val); + conf->cache.xdp_tcp = running_xdp_tcp; + conf->cache.xdp_route_check = running_route_check; val = conf_get(conf, C_CTL, C_TIMEOUT); diff --git a/src/knot/conf/base.h b/src/knot/conf/base.h index 899dcc0b3..d87ed1223 100644 --- a/src/knot/conf/base.h +++ b/src/knot/conf/base.h @@ -121,10 +121,11 @@ typedef struct { size_t srv_xdp_threads; size_t srv_bg_threads; size_t srv_tcp_max_clients; - uint32_t xdp_tcp_max_conns; - size_t xdp_tcp_inbufs_size; + size_t xdp_tcp_max_clients; + size_t xdp_tcp_inbuf_max_size; uint32_t xdp_tcp_idle_close; uint32_t xdp_tcp_idle_reset; + bool xdp_tcp; bool xdp_route_check; int ctl_timeout; conf_val_t srv_nsid; diff --git a/src/knot/conf/schema.c b/src/knot/conf/schema.c index 9312ec7df..3de4ebd62 100644 --- a/src/knot/conf/schema.c +++ b/src/knot/conf/schema.c @@ -227,9 +227,9 @@ static const yp_item_t desc_server[] = { static const yp_item_t desc_xdp[] = { { C_LISTEN, YP_TADDR, YP_VADDR = { 53 }, YP_FMULTI, { check_xdp } }, - { C_XDP_TCP, YP_TBOOL, YP_VNONE }, + { C_TCP, YP_TBOOL, YP_VNONE }, { C_TCP_MAX_CLIENTS, YP_TINT, YP_VINT = { 0, INT32_MAX, 1000000 } }, - { C_TCP_MAX_INBUFS, YP_TINT, YP_VINT = { MEGA(1), SSIZE_MAX, MEGA(100), YP_SSIZE } }, + { C_TCP_INBUF_MAX_SIZE, YP_TINT, YP_VINT = { MEGA(1), SSIZE_MAX, MEGA(100), YP_SSIZE } }, { C_TCP_IDLE_CLOSE, YP_TINT, YP_VINT = { 1, INT32_MAX, 10, YP_STIME } }, { C_TCP_IDLE_RESET, YP_TINT, YP_VINT = { 1, INT32_MAX, 20, YP_STIME } }, { C_ROUTE_CHECK, YP_TBOOL, YP_VNONE }, diff --git a/src/knot/conf/schema.h b/src/knot/conf/schema.h index efa9b61a5..e4ffb28db 100644 --- a/src/knot/conf/schema.h +++ b/src/knot/conf/schema.h @@ -31,8 +31,8 @@ #define C_BACKEND "\x07""backend" #define C_BG_WORKERS "\x12""background-workers" #define C_BLOCK_NOTIFY_XFR "\x1B""block-notify-after-transfer" -#define C_CATALOG_DB "\x0A""catalog-db" -#define C_CATALOG_DB_MAX_SIZE "\x13""catalog-db-max-size" +#define C_CATALOG_DB "\x0A""catalog-db" +#define C_CATALOG_DB_MAX_SIZE "\x13""catalog-db-max-size" #define C_CATALOG_ROLE "\x0C""catalog-role" #define C_CATALOG_TPL "\x10""catalog-template" #define C_CATALOG_ZONE "\x0C""catalog-zone" @@ -110,13 +110,14 @@ #define C_STATS "\x0A""statistics" #define C_STORAGE "\x07""storage" #define C_TARGET "\x06""target" +#define C_TCP "\x03""tcp" #define C_TCP_FASTOPEN "\x0C""tcp-fastopen" -#define C_TCP_IDLE_CLOSE "\x0E""tcp-idle-close" -#define C_TCP_IDLE_RESET "\x0E""tcp-idle-reset" +#define C_TCP_IDLE_CLOSE "\x16""tcp-idle-close-timeout" +#define C_TCP_IDLE_RESET "\x16""tcp-idle-reset-timeout" #define C_TCP_IDLE_TIMEOUT "\x10""tcp-idle-timeout" +#define C_TCP_INBUF_MAX_SIZE "\x12""tcp-inbuf-max-size" #define C_TCP_IO_TIMEOUT "\x0E""tcp-io-timeout" #define C_TCP_MAX_CLIENTS "\x0F""tcp-max-clients" -#define C_TCP_MAX_INBUFS "\x0E""tcp-max-inbufs" #define C_TCP_REUSEPORT "\x0D""tcp-reuseport" #define C_TCP_RMT_IO_TIMEOUT "\x15""tcp-remote-io-timeout" #define C_TCP_WORKERS "\x0B""tcp-workers" @@ -138,7 +139,6 @@ #define C_VERSION "\x07""version" #define C_VIA "\x03""via" #define C_XDP "\x03""xdp" -#define C_XDP_TCP "\x07""xdp-tcp" #define C_ZONE "\x04""zone" #define C_ZONEFILE_LOAD "\x0D""zonefile-load" #define C_ZONEFILE_SYNC "\x0D""zonefile-sync" diff --git a/src/knot/conf/tools.c b/src/knot/conf/tools.c index 8e1f057d9..125f01454 100644 --- a/src/knot/conf/tools.c +++ b/src/knot/conf/tools.c @@ -418,11 +418,13 @@ int check_server( { conf_val_t listen = conf_get_txn(args->extra->conf, args->extra->txn, C_SRV, C_LISTEN); - conf_val_t xdp = conf_get_txn(args->extra->conf, args->extra->txn, C_SRV, - C_LISTEN_XDP); + conf_val_t xdp = conf_get_txn(args->extra->conf, args->extra->txn, C_XDP, + C_LISTEN); + conf_val_t tcp = conf_get_txn(args->extra->conf, args->extra->txn, C_XDP, + C_TCP); if (xdp.code == KNOT_EOK) { - if (listen.code != KNOT_EOK) { - CONF_LOG(LOG_WARNING, "unable to process TCP queries due to XDP-only interfaces"); + if (listen.code != KNOT_EOK && tcp.code != KNOT_EOK) { + CONF_LOG(LOG_WARNING, "unavailable TCP processing"); } check_mtu(args, &xdp); } diff --git a/src/knot/server/server.c b/src/knot/server/server.c index 3339f9b87..9d64a95a7 100644 --- a/src/knot/server/server.c +++ b/src/knot/server/server.c @@ -559,17 +559,15 @@ static int configure_sockets(conf_t *conf, server_t *s) free(rundir); /* XDP sockets. */ + bool xdp_tcp = conf->cache.xdp_tcp; bool route_check = conf->cache.xdp_route_check; unsigned thread_id = s->handlers[IO_UDP].handler.unit->size + s->handlers[IO_TCP].handler.unit->size; while (lisxdp_val.code == KNOT_EOK) { - conf_val_t xdp_tcp_val = conf_get(conf, C_XDP, C_XDP_TCP); - bool xdp_tcp = conf_bool(&xdp_tcp_val); - struct sockaddr_storage addr = conf_addr(&lisxdp_val, NULL); char addr_str[SOCKADDR_STRLEN] = { 0 }; sockaddr_tostr(addr_str, sizeof(addr_str), &addr); - log_info("binding to XDP interface %s%s", addr_str, xdp_tcp ? " with TCP" : ""); + log_info("binding to XDP interface %s for UDP%s", addr_str, xdp_tcp ? " and TCP" : ""); iface_t *new_if = server_init_xdp_iface(&addr, route_check, &thread_id, xdp_tcp); if (new_if == NULL) { @@ -919,6 +917,7 @@ static void warn_server_reconfigure(conf_t *conf, server_t *server) static bool warn_tcp = true; static bool warn_bg = true; static bool warn_listen = true; + static bool warn_xdp_tcp = true; static bool warn_route_check = true; if (warn_tcp_reuseport && conf->cache.srv_tcp_reuseport != conf_get_bool(conf, C_SRV, C_TCP_REUSEPORT)) { @@ -951,6 +950,11 @@ static void warn_server_reconfigure(conf_t *conf, server_t *server) warn_listen = false; } + if (warn_xdp_tcp && conf->cache.xdp_tcp != conf_get_bool(conf, C_XDP, C_TCP)) { + log_warning(msg, &C_TCP[1]); + warn_xdp_tcp = false; + } + if (warn_route_check && conf->cache.xdp_route_check != conf_get_bool(conf, C_XDP, C_ROUTE_CHECK)) { log_warning(msg, &C_ROUTE_CHECK[1]); warn_route_check = false; diff --git a/src/knot/server/xdp-handler.c b/src/knot/server/xdp-handler.c index f8cc52fec..bd2ff092a 100644 --- a/src/knot/server/xdp-handler.c +++ b/src/knot/server/xdp-handler.c @@ -37,8 +37,8 @@ typedef struct xdp_handle_ctx { uint32_t msg_udp_count; knot_tcp_table_t *tcp_table; - uint32_t tcp_max_conns; - size_t tcp_inbufs_size; + size_t tcp_max_conns; + size_t tcp_max_inbufs; uint32_t tcp_idle_close; uint32_t tcp_idle_reset; } xdp_handle_ctx_t; @@ -62,10 +62,10 @@ void xdp_handle_reconfigure(xdp_handle_ctx_t *ctx) { rcu_read_lock(); conf_t *pconf = conf(); - ctx->tcp_max_conns = pconf->cache.xdp_tcp_max_conns / pconf->cache.srv_xdp_threads; - ctx->tcp_inbufs_size = pconf->cache.xdp_tcp_inbufs_size / pconf->cache.srv_xdp_threads; - ctx->tcp_idle_close = pconf->cache.xdp_tcp_idle_close * 1000000; // conf:secs -> tcp:usecs - ctx->tcp_idle_reset = pconf->cache.xdp_tcp_idle_reset * 1000000; + ctx->tcp_max_conns = pconf->cache.xdp_tcp_max_clients / pconf->cache.srv_xdp_threads; + ctx->tcp_max_inbufs = pconf->cache.xdp_tcp_inbuf_max_size / pconf->cache.srv_xdp_threads; + ctx->tcp_idle_close = pconf->cache.xdp_tcp_idle_close * 1000000; // conf:secs -> tcp:usecs + ctx->tcp_idle_reset = pconf->cache.xdp_tcp_idle_reset * 1000000; rcu_read_unlock(); } @@ -242,7 +242,7 @@ int xdp_handle_timeout(xdp_handle_ctx_t *ctx, knot_xdp_socket_t *xdp_sock) do { ret = knot_xdp_tcp_timeout(ctx->tcp_table, xdp_sock, 20, ctx->tcp_idle_close, ctx->tcp_idle_reset, overweight(ctx->tcp_table->usage, ctx->tcp_max_conns), - overweight(ctx->tcp_table->inbufs_total, ctx->tcp_inbufs_size), + overweight(ctx->tcp_table->inbufs_total, ctx->tcp_max_inbufs), &last_close, &last_reset); } while (last_reset > 0 && ret == KNOT_EOK); diff --git a/tests/knot/test_confio.c b/tests/knot/test_confio.c index 560025c19..eb459bcc5 100644 --- a/tests/knot/test_confio.c +++ b/tests/knot/test_confio.c @@ -945,11 +945,12 @@ static const yp_item_t desc_server[] = { }; static const yp_item_t desc_xdp[] = { - { C_TCP_MAX_CLIENTS, YP_TINT, YP_VNONE }, - { C_TCP_MAX_INBUFS, YP_TINT, YP_VNONE }, - { C_TCP_IDLE_CLOSE, YP_TINT, YP_VNONE }, - { C_TCP_IDLE_RESET, YP_TINT, YP_VNONE }, - { C_ROUTE_CHECK, YP_TBOOL, YP_VNONE }, + { C_TCP, YP_TBOOL, YP_VNONE }, + { C_TCP_MAX_CLIENTS, YP_TINT, YP_VNONE }, + { C_TCP_INBUF_MAX_SIZE, YP_TINT, YP_VNONE }, + { C_TCP_IDLE_CLOSE, YP_TINT, YP_VNONE }, + { C_TCP_IDLE_RESET, YP_TINT, YP_VNONE }, + { C_ROUTE_CHECK, YP_TBOOL, YP_VNONE }, { NULL } };