From 61cf8dd6b190dfdb3efabb5f1df0f35bd7f26f99 Mon Sep 17 00:00:00 2001 From: vsaranchuk Date: Mon, 18 May 2026 09:55:21 +0200 Subject: [PATCH] Fix Keycloak Connection Timeout Issue to Prevent Hanging Connections Closes #47174 Signed-off-by: Vadym Saranchuk Signed-off-by: vsaranchuk Signed-off-by: Alexander Schwartz Co-authored-by: Vadym Saranchuk Co-authored-by: Alexander Schwartz --- .../topics/changes/changes-26_7_0.adoc | 4 + ...TimeoutOnConnectionAcquireInterceptor.java | 95 +++++++++++++++++++ .../mappers/DatabasePropertyMappers.java | 76 +++++++++++---- .../models/utils/KeycloakModelUtils.java | 12 +++ 4 files changed, 169 insertions(+), 18 deletions(-) create mode 100644 quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/UpdateSocketTimeoutOnConnectionAcquireInterceptor.java diff --git a/docs/documentation/upgrading/topics/changes/changes-26_7_0.adoc b/docs/documentation/upgrading/topics/changes/changes-26_7_0.adoc index c662c15d9d1..39b584c2b35 100644 --- a/docs/documentation/upgrading/topics/changes/changes-26_7_0.adoc +++ b/docs/documentation/upgrading/topics/changes/changes-26_7_0.adoc @@ -47,6 +47,10 @@ that this option is deprecated and exists mostly for the backwards compatibility For the details, see the link:{adminguide_link}[{adminguide_name}]. +=== Socket timeouts and query timeouts for database connections + +For all database connections, {project_name} now sets a socket read timeout matching the respective transaction timeout to prevent long hanging threads. For PostgreSQL databases, it additionally sets a query timeout. + === Configure TOTP and Update password required actions moved after Verify Email In relation to the previous point, the required actions *Configure OTP* and *Update password* are moved in the order of required actions after *Verify Email* diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/UpdateSocketTimeoutOnConnectionAcquireInterceptor.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/UpdateSocketTimeoutOnConnectionAcquireInterceptor.java new file mode 100644 index 00000000000..a01ced2d94c --- /dev/null +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/UpdateSocketTimeoutOnConnectionAcquireInterceptor.java @@ -0,0 +1,95 @@ +/* + * Copyright 2026 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.quarkus.runtime.configuration; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import jakarta.enterprise.context.ApplicationScoped; + +import org.keycloak.config.TransactionOptions; +import org.keycloak.models.utils.KeycloakModelUtils; + +import io.agroal.api.AgroalPoolInterceptor; +import io.agroal.pool.wrapper.ConnectionWrapper; +import io.quarkus.runtime.configuration.DurationConverter; +import org.jspecify.annotations.NonNull; +import org.postgresql.PGConnection; + +/** + * When acquiring a database connection, set the socket timeout to the same value as the transaction timeout. + * This will shorten the time between the transaction manager aborting the transaction but not cancelling the statement, + * and finally freeing the connection and the Java thread as no SQL statement will run longer than the socket timeout. + */ +@ApplicationScoped +public class UpdateSocketTimeoutOnConnectionAcquireInterceptor implements AgroalPoolInterceptor { + + // Executor is not closed during shutdown to avoid interfering with shutting down database connections + private final Executor executor; + private final Integer transactionDefaultTimeout; + private Class pgClass; + + public UpdateSocketTimeoutOnConnectionAcquireInterceptor() { + this.executor = Executors + .newSingleThreadExecutor(new InternalThreadFactory()); + this.transactionDefaultTimeout = + Long.valueOf(DurationConverter.parseDuration( + Configuration.getConfigValue(TransactionOptions.TRANSACTION_DEFAULT_TIMEOUT).getValue() + ).toMillis()).intValue(); + try { + pgClass = Class.forName("org.postgresql.PGConnection"); + } catch (ClassNotFoundException ex) { + pgClass = null; + } + } + + @Override + public void onConnectionAcquire(Connection connection) { + try { + Optional timeout = KeycloakModelUtils.getTransactionLimit(); + int timeoutMillis = timeout.map(integer -> integer * 1000).orElse(transactionDefaultTimeout); + if (pgClass != null && connection instanceof ConnectionWrapper wrapper && wrapper.isWrapperFor(pgClass)) { + // PostgreSQL allows for a more graceful termination than the read timeout. + // That would then be only the last resort on network problems. + wrapper.unwrap(PGConnection.class).setQueryTimeout((int) TimeUnit.MILLISECONDS.toSeconds(timeoutMillis)); + timeoutMillis += 1000; + } + connection.setNetworkTimeout(executor, timeoutMillis); + } catch (SQLException e) { + throw new IllegalStateException("Can't set timeouts for connection", e); + } + } + + private static class InternalThreadFactory implements ThreadFactory { + + private static final String THREAD_NAME = "jdbc-network-timeout"; + + @Override + public Thread newThread(@NonNull Runnable runnable) { + Thread thread = new Thread(runnable); + thread.setName(THREAD_NAME); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/DatabasePropertyMappers.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/DatabasePropertyMappers.java index 080c461cad6..93d914b7fb3 100644 --- a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/DatabasePropertyMappers.java +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/DatabasePropertyMappers.java @@ -56,6 +56,7 @@ public final class DatabasePropertyMappers implements PropertyMapperGrouping { public static final String PG_TARGET_SERVER_TYPE = "quarkus.datasource.jdbc.additional-jdbc-properties.targetServerType"; public static final String MSSQL_SEND_STRING_PARAMETER_AS_UNICODE = "quarkus.datasource.jdbc.additional-jdbc-properties.sendStringParametersAsUnicode"; public static final String CONNECT_TIMEOUT = "quarkus.datasource.jdbc.additional-jdbc-properties.connectTimeout"; + public static final String SOCKET_TIMEOUT = "quarkus.datasource.jdbc.additional-jdbc-properties.socketTimeout"; public static final String ORACLEDB_CONNECT_TIMEOUT = "quarkus.datasource.jdbc.additional-jdbc-properties.oracle.net.CONNECT_TIMEOUT"; public static final String MSSQL_CONNECT_TIMEOUT = "quarkus.datasource.jdbc.additional-jdbc-properties.loginTimeout"; public static final String JDBC_LOGIN_TIMEOUT = "quarkus.datasource.jdbc.login-timeout"; @@ -112,6 +113,15 @@ public final class DatabasePropertyMappers implements PropertyMapperGrouping { .to(MSSQL_CONNECT_TIMEOUT) .mapFrom(DatabaseOptions.DB_CONNECT_TIMEOUT, getConnectTimeout(EnumSet.of(Database.Vendor.MSSQL), "loginTimeout")) .build(), + /* For MySQL based databases, setting the login timeout is not sufficient as there are some additional SQL statements running + directly after the login. Once the connection is later acquired for a transaction, the UpdateSocketTimeoutOnConnectionAcquireInterceptor + will then later overwrite the socket timeout. + See https://github.com/keycloak/keycloak/issues/47174 for the discussion. + */ + fromOption(DatabaseOptions.DB_CONNECT_TIMEOUT) + .to(SOCKET_TIMEOUT) + .mapFrom(DatabaseOptions.DB_CONNECT_TIMEOUT, getSocketTimeout(EnumSet.of(Database.Vendor.MYSQL, Database.Vendor.MARIADB, Database.Vendor.TIDB), "socketTimeout")) + .build(), fromOption(DatabaseOptions.DB_URL_HOST) .paramLabel("hostname") .build(), @@ -327,33 +337,63 @@ public final class DatabasePropertyMappers implements PropertyMapperGrouping { String db = getDatasourceOptionValue(DB, datasource).orElse(null); Database.Vendor vendor = Database.getVendor(db).orElse(null); - if (!validForVendors.contains(vendor)) { - // this jdbc property is not for this vendor - return null; - } - - String dbDriver = getDatasourceOptionValue(DatabaseOptions.DB_DRIVER, datasource).orElse(null); - if (!Objects.equals(Database.getDriver(db, true).orElse(null), dbDriver) && - !Objects.equals(Database.getDriver(db, false).orElse(null), dbDriver)) { - // Custom JDBC driver (e.g. AWS JDBC Wrapper) — do not inject defaults - return null; - } - - String dbUrl = findDatabaseUrl(datasource).orElse(""); - String dbUrlProperties = getDatasourceOptionValue(DatabaseOptions.DB_URL_PROPERTIES, datasource).orElse(""); - - // Property already set explicitly by the user — do not override - if (dbUrl.contains(timeoutProperty) || dbUrlProperties.contains(timeoutProperty)) { + if (checkSettingsAndVendor(validForVendors, timeoutProperty, datasource, vendor, db)) { return null; } if (vendor == Vendor.MSSQL || vendor == Vendor.POSTGRES) { return durationToSeconds(value); } - return durationToMillis(value); + if (vendor == Vendor.MYSQL || vendor == Vendor.MARIADB || vendor == Vendor.ORACLE || vendor == Vendor.TIDB) { + return durationToMillis(value); + } + + // We don't know if it is seconds or milliseconds for other databases. + throw new IllegalArgumentException("Vendor " + vendor + " not supported for socket timeout calculation"); }; } + private static ValueMapper getSocketTimeout(Collection validForVendors, String timeoutProperty) { + return (String datasource, String value, ConfigSourceInterceptorContext context) -> { + String db = getDatasourceOptionValue(DB, datasource).orElse(null); + Database.Vendor vendor = Database.getVendor(db).orElse(null); + + if (checkSettingsAndVendor(validForVendors, timeoutProperty, datasource, vendor, db)) { + return null; + } + + if (vendor == Vendor.MYSQL || vendor == Vendor.MARIADB || vendor == Vendor.TIDB) { + return durationToMillis(value); + } + + // We don't know if it is seconds or milliseconds for other database. + throw new IllegalArgumentException("Vendor " + vendor + " not supported for socket timeout calculation"); + }; + } + + private static boolean checkSettingsAndVendor(Collection validForVendors, String timeoutProperty, String datasource, Vendor vendor, String db) { + if (!validForVendors.contains(vendor)) { + // this jdbc property is not for this vendor + return true; + } + + String dbDriver = getDatasourceOptionValue(DatabaseOptions.DB_DRIVER, datasource).orElse(null); + if (!Objects.equals(Database.getDriver(db, true).orElse(null), dbDriver) && + !Objects.equals(Database.getDriver(db, false).orElse(null), dbDriver)) { + // Custom JDBC driver (e.g. AWS JDBC Wrapper) — do not inject defaults + return true; + } + + String dbUrl = findDatabaseUrl(datasource).orElse(""); + String dbUrlProperties = getDatasourceOptionValue(DatabaseOptions.DB_URL_PROPERTIES, datasource).orElse(""); + + // Property already set explicitly by the user — do not override + if (dbUrl.contains(timeoutProperty) || dbUrlProperties.contains(timeoutProperty)) { + return true; + } + return false; + } + private static String durationToMillis(String value) { return String.valueOf(DurationConverter.parseDuration(value).toMillis()); } diff --git a/server-spi-private/src/main/java/org/keycloak/models/utils/KeycloakModelUtils.java b/server-spi-private/src/main/java/org/keycloak/models/utils/KeycloakModelUtils.java index 1323c496d8f..f59c41451e9 100755 --- a/server-spi-private/src/main/java/org/keycloak/models/utils/KeycloakModelUtils.java +++ b/server-spi-private/src/main/java/org/keycloak/models/utils/KeycloakModelUtils.java @@ -121,6 +121,8 @@ public final class KeycloakModelUtils { public static final int DEFAULT_RSA_KEY_SIZE = 4096; public static final int DEFAULT_CERTIFICATE_VALIDITY_YEARS = 3; + private static final ThreadLocal timeouts = new ThreadLocal(); + private KeycloakModelUtils() { } @@ -501,6 +503,12 @@ public final class KeycloakModelUtils { try { // If timeout is set to 0, reset to default transaction timeout lookup.getTransactionManager().setTransactionTimeout(timeoutInSeconds); + + if (timeoutInSeconds == 0) { + timeouts.remove(); + } else { + timeouts.set(timeoutInSeconds); + } } catch (SystemException e) { // Shouldn't happen for Wildfly transaction manager throw new RuntimeException(e); @@ -509,6 +517,10 @@ public final class KeycloakModelUtils { } } + public static Optional getTransactionLimit() { + return Optional.ofNullable(timeouts.get()); + } + public static Function componentModelGetter(String realmId, String componentId) { return factory -> getComponentModel(factory, realmId, componentId); }