upstream/haproxy
1
0
Fork 0
mirror of https://github.com/haproxy/haproxy.git synced 2026-06-13 10:50:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

MEDIUM: cpu-topo: Add a new "max-threads-per-group" global keyword

Browse source 
Add a new global keyword, max-threads-per-group. It sets the maximum number of
threads a thread group can contain. Unless the number of thread groups
is fixed with "thread-groups", haproxy will just create more thread
groups as needed.
The default and maximum value is 64.
This commit is contained in:
Olivier Houchard 2025-12-08 23:13:19 +01:00 committed by Olivier Houchard
parent 3865f6c5c6
commit 7e22d9c484
5 changed files with 65 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
doc/configuration.txt
View file

@ -1787,6 +1787,7 @@ The following keywords are supported in the "global" section :
- lua-load - lua-load
- lua-load-per-thread - lua-load-per-thread
- lua-prepend-path - lua-prepend-path
- max-thread-per-group
- mworker-max-reloads - mworker-max-reloads
- nbthread - nbthread
- node - node
@ -2997,6 +2998,14 @@ master-worker no-exit-on-failure
it is only meant for debugging and could put the master process in an it is only meant for debugging and could put the master process in an
abnormal state. abnormal state.
max-threads-per-group <number>
Defines the maximum number of threads in a thread group. Unless the number
of thread groups is fixed with the thread-groups directive, haproxy will
create more thread groups if needed. The default and maximum value is 64.
Having a lower value means more groups will potentially be created, which
can help improve performances, as a number of data structures are per
thread group, and that will mean less contention
mworker-max-reloads <number> mworker-max-reloads <number>
In master-worker mode, this option limits the number of time a worker can In master-worker mode, this option limits the number of time a worker can
survive to a reload. If the worker did not leave after a reload, once its survive to a reload. If the worker did not leave after a reload, once its

1
include/haproxy/global-t.h
View file

@ -261,6 +261,7 @@ struct global {
unsigned int req_count; /* request counter (HTTP or TCP session) for logs and unique_id */ unsigned int req_count; /* request counter (HTTP or TCP session) for logs and unique_id */
int last_checks; int last_checks;
uint32_t anon_key; uint32_t anon_key;
int maxthrpertgroup; /* Maximum number of threads per thread group */
/* leave this at the end to make sure we don't share this cache line by accident */ /* leave this at the end to make sure we don't share this cache line by accident */
ALWAYS_ALIGN(64); ALWAYS_ALIGN(64);

12
src/cpu_topo.c
View file

@ -1255,12 +1255,12 @@ static int cpu_policy_group_by_cluster(int policy, int tmin, int tmax, int gmin,
* CPUs but enough groups left, we'll try to make more smaller * CPUs but enough groups left, we'll try to make more smaller
* groups, of the closest size each. * groups, of the closest size each.
*/ */
nb_grp = (cpu_count + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP; nb_grp = (cpu_count + global.maxthrpertgroup - 1) / global.maxthrpertgroup;
if (nb_grp > MAX_TGROUPS - global.nbtgroups) if (nb_grp > MAX_TGROUPS - global.nbtgroups)
nb_grp = MAX_TGROUPS - global.nbtgroups; nb_grp = MAX_TGROUPS - global.nbtgroups;
thr_per_grp = (cpu_count + nb_grp - 1) / nb_grp; thr_per_grp = (cpu_count + nb_grp - 1) / nb_grp;
if (thr_per_grp > MAX_THREADS_PER_GROUP) if (thr_per_grp > global.maxthrpertgroup)
thr_per_grp = MAX_THREADS_PER_GROUP; thr_per_grp = global.maxthrpertgroup;
while (nb_grp && cpu_count > 0) { while (nb_grp && cpu_count > 0) {
/* create at most thr_per_grp threads */ /* create at most thr_per_grp threads */
@ -1414,12 +1414,12 @@ static int cpu_policy_group_by_ccx(int policy, int tmin, int tmax, int gmin, int
* CPUs but enough groups left, we'll try to make more smaller * CPUs but enough groups left, we'll try to make more smaller
* groups, of the closest size each. * groups, of the closest size each.
*/ */
nb_grp = (cpu_count + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP; nb_grp = (cpu_count + global.maxthrpertgroup - 1) / global.maxthrpertgroup;
if (nb_grp > MAX_TGROUPS - global.nbtgroups) if (nb_grp > MAX_TGROUPS - global.nbtgroups)
nb_grp = MAX_TGROUPS - global.nbtgroups; nb_grp = MAX_TGROUPS - global.nbtgroups;
thr_per_grp = (cpu_count + nb_grp - 1) / nb_grp; thr_per_grp = (cpu_count + nb_grp - 1) / nb_grp;
if (thr_per_grp > MAX_THREADS_PER_GROUP) if (thr_per_grp > global.maxthrpertgroup)
thr_per_grp = MAX_THREADS_PER_GROUP; thr_per_grp = global.maxthrpertgroup;
while (nb_grp && cpu_count > 0) { while (nb_grp && cpu_count > 0) {
/* create at most thr_per_grp threads */ /* create at most thr_per_grp threads */

10
src/listener.c
View file

@ -229,7 +229,7 @@ REGISTER_POST_DEINIT(accept_queue_deinit);
*/ */
int li_init_per_thr(struct listener *li) int li_init_per_thr(struct listener *li)
{ {
int nbthr = MIN(global.nbthread, MAX_THREADS_PER_GROUP); int nbthr = MIN(global.nbthread, global.maxthrpertgroup);
int i; int i;
/* allocate per-thread elements for listener */ /* allocate per-thread elements for listener */
@ -1394,7 +1394,7 @@ void listener_accept(struct listener *l)
/* no more threads here, switch to /* no more threads here, switch to
* last thread of previous group. * last thread of previous group.
*/ */
t2 = MAX_THREADS_PER_GROUP - 1; t2 = global.maxthrpertgroup - 1;
if (l->rx.shard_info) if (l->rx.shard_info)
r2--; r2--;
/* loop again */ /* loop again */
@ -1456,10 +1456,10 @@ void listener_accept(struct listener *l)
new_li = l->rx.shard_info->members[r1]->owner; new_li = l->rx.shard_info->members[r1]->owner;
t2--; t2--;
if (t2 >= MAX_THREADS_PER_GROUP) { if (t2 >= global.maxthrpertgroup) {
if (l->rx.shard_info) if (l->rx.shard_info)
r2--; r2--;
t2 = MAX_THREADS_PER_GROUP - 1; t2 = global.maxthrpertgroup - 1;
} }
} }
else if (q1 - q2 > 0) { else if (q1 - q2 > 0) {
@ -1480,7 +1480,7 @@ void listener_accept(struct listener *l)
new_li = l->rx.shard_info->members[r1]->owner; new_li = l->rx.shard_info->members[r1]->owner;
updt_t1: updt_t1:
t1++; t1++;
if (t1 >= MAX_THREADS_PER_GROUP) { if (t1 >= global.maxthrpertgroup) {
if (l->rx.shard_info) if (l->rx.shard_info)
r1++; r1++;
t1 = 0; t1 = 0;

62
src/thread.c
View file

@ -1415,7 +1415,7 @@ int thread_map_to_groups()
*/ */
q = ut / ug; q = ut / ug;
r = ut % ug; r = ut % ug;
if ((q + !!r) > MAX_THREADS_PER_GROUP) { if ((q + !!r) > global.maxthrpertgroup) {
ha_alert("Too many remaining unassigned threads (%d) for thread groups (%d). Please increase thread-groups or make sure to keep thread numbers contiguous\n", ut, ug); ha_alert("Too many remaining unassigned threads (%d) for thread groups (%d). Please increase thread-groups or make sure to keep thread numbers contiguous\n", ut, ug);
return -1; return -1;
} }
@ -1645,6 +1645,9 @@ void thread_detect_count(void)
if (global.nbtgroups) if (global.nbtgroups)
grp_min = grp_max = global.nbtgroups; grp_min = grp_max = global.nbtgroups;
if (!global.maxthrpertgroup)
global.maxthrpertgroup = MAX_THREADS_PER_GROUP;
#if defined(USE_THREAD) #if defined(USE_THREAD)
/* Adjust to boot settings if not forced */ /* Adjust to boot settings if not forced */
if (thr_min <= thread_cpus_enabled_at_boot && thread_cpus_enabled_at_boot < thr_max) if (thr_min <= thread_cpus_enabled_at_boot && thread_cpus_enabled_at_boot < thr_max)
@ -1668,13 +1671,13 @@ void thread_detect_count(void)
if (thr_min < grp_min && thr_max >= grp_min) if (thr_min < grp_min && thr_max >= grp_min)
thr_min = grp_min; thr_min = grp_min;
if (thr_min <= MAX_THREADS_PER_GROUP * grp_max && if (thr_min <= global.maxthrpertgroup * grp_max &&
thr_max > MAX_THREADS_PER_GROUP * grp_max) thr_max > global.maxthrpertgroup * grp_max)
thr_max = MAX_THREADS_PER_GROUP * grp_max; thr_max = global.maxthrpertgroup * grp_max;
if (grp_min < (thr_min + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP && if (grp_min < (thr_min + global.maxthrpertgroup - 1) / global.maxthrpertgroup &&
grp_max >= (thr_min + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP) grp_max >= (thr_min + global.maxthrpertgroup - 1) / global.maxthrpertgroup)
grp_min = (thr_min + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP; grp_min = (thr_min + global.maxthrpertgroup - 1) / global.maxthrpertgroup;
if (grp_max > thr_max && grp_min <= thr_max) if (grp_max > thr_max && grp_min <= thr_max)
grp_max = thr_max; grp_max = thr_max;
@ -1738,10 +1741,10 @@ void thread_detect_count(void)
if (!global.nbtgroups) if (!global.nbtgroups)
global.nbtgroups = 1; global.nbtgroups = 1;
if (global.nbthread > MAX_THREADS_PER_GROUP * global.nbtgroups) { if (global.nbthread > global.maxthrpertgroup * global.nbtgroups) {
ha_diag_warning("nbthread too large or not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group and %d groups). Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n", ha_diag_warning("nbthread too large or not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group and %d groups). Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n",
global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP, MAX_TGROUPS); global.nbthread, global.maxthrpertgroup * global.nbtgroups, global.maxthrpertgroup, MAX_TGROUPS);
global.nbthread = MAX_THREADS_PER_GROUP * global.nbtgroups; global.nbthread = global.maxthrpertgroup * global.nbtgroups;
} }
return; return;
} }
@ -1871,7 +1874,7 @@ int parse_thread_set(const char *arg, struct thread_set *ts, char **err)
if (!*set) { if (!*set) {
/* empty set sets no restriction */ /* empty set sets no restriction */
min = 1; min = 1;
max = is_rel ? MAX_THREADS_PER_GROUP : MAX_THREADS; max = is_rel ? global.maxthrpertgroup : MAX_THREADS;
} }
else { else {
if (sep != set && *sep && *sep != '-' && *sep != ',') { if (sep != set && *sep && *sep != '-' && *sep != ',') {
@ -1899,9 +1902,9 @@ int parse_thread_set(const char *arg, struct thread_set *ts, char **err)
max = min = 0; // throw an error below max = min = 0; // throw an error below
} }
if (min < 1 || min > MAX_THREADS || (is_rel && min > MAX_THREADS_PER_GROUP)) { if (min < 1 || min > MAX_THREADS || (is_rel && min > global.maxthrpertgroup)) {
memprintf(err, "invalid first thread number '%s', permitted range is 1..%d, or 'all', 'odd', 'even'.", memprintf(err, "invalid first thread number '%s', permitted range is 1..%d, or 'all', 'odd', 'even'.",
set, is_rel ? MAX_THREADS_PER_GROUP : MAX_THREADS); set, is_rel ? global.maxthrpertgroup : MAX_THREADS);
return -1; return -1;
} }
@ -1918,15 +1921,15 @@ int parse_thread_set(const char *arg, struct thread_set *ts, char **err)
v = atoi(set); v = atoi(set);
if (sep == set) { // no digit: to the max if (sep == set) { // no digit: to the max
max = is_rel ? MAX_THREADS_PER_GROUP : MAX_THREADS; max = is_rel ? global.maxthrpertgroup : MAX_THREADS;
if (*sep && *sep != ',') if (*sep && *sep != ',')
max = 0; // throw an error below max = 0; // throw an error below
} else } else
max = v; max = v;
if (max < 1 || max > MAX_THREADS || (is_rel && max > MAX_THREADS_PER_GROUP)) { if (max < 1 || max > MAX_THREADS || (is_rel && max > global.maxthrpertgroup)) {
memprintf(err, "invalid last thread number '%s', permitted range is 1..%d.", memprintf(err, "invalid last thread number '%s', permitted range is 1..%d.",
set, is_rel ? MAX_THREADS_PER_GROUP : MAX_THREADS); set, is_rel ? global.maxthrpertgroup : MAX_THREADS);
return -1; return -1;
} }
} }
@ -2138,14 +2141,36 @@ static int cfg_parse_thread_group(char **args, int section_type, struct proxy *c
return -1; return -1;
} }
if (ha_tgroup_info[tgroup-1].count > MAX_THREADS_PER_GROUP) { if (ha_tgroup_info[tgroup-1].count > global.maxthrpertgroup) {
memprintf(err, "'%s %ld' assigned too many threads (%d, max=%d)", args[0], tgroup, tot, MAX_THREADS_PER_GROUP); memprintf(err, "'%s %ld' assigned too many threads (%d, max=%d)", args[0], tgroup, tot, global.maxthrpertgroup);
return -1; return -1;
} }
return 0; return 0;
} }
/* Parse the "max-threads-per-group" global directive, which indicates the
* maximum number of thread to have in one thread group
*/
static int cfg_parse_maxthreadpertgroup(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
{
long maxthrpertg;
char *errptr;
if (too_many_args(1, args, err, NULL))
return -1;
maxthrpertg = strtol(args[1], &errptr, 10);
if (!*args[1] || *errptr || maxthrpertg < 0 || maxthrpertg > MAX_THREADS_PER_GROUP) {
memprintf(err, "'%s' value must be an integer between 1 and %d, got '%s'", args[0], MAX_THREADS_PER_GROUP, args[1]);
return -1;
}
global.maxthrpertgroup = maxthrpertg;
return 0;
}
/* Parse the "thread-groups" global directive, which takes an integer argument /* Parse the "thread-groups" global directive, which takes an integer argument
* that contains the desired number of thread groups. * that contains the desired number of thread groups.
*/ */
@ -2196,6 +2221,7 @@ static struct cfg_kw_list cfg_kws = {ILH, {
{ CFG_GLOBAL, "nbthread", cfg_parse_nbthread, 0 }, { CFG_GLOBAL, "nbthread", cfg_parse_nbthread, 0 },
{ CFG_GLOBAL, "thread-group", cfg_parse_thread_group, 0 }, { CFG_GLOBAL, "thread-group", cfg_parse_thread_group, 0 },
{ CFG_GLOBAL, "thread-groups", cfg_parse_thread_groups, 0 }, { CFG_GLOBAL, "thread-groups", cfg_parse_thread_groups, 0 },
{ CFG_GLOBAL, "max-threads-per-group", cfg_parse_maxthreadpertgroup, 0 },
{ 0, NULL, NULL } { 0, NULL, NULL }
}}; }};