JGroups certificate rotation

Closes #37316

Signed-off-by: Pedro Ruivo <pruivo@redhat.com>
Signed-off-by: Alexander Schwartz <aschwart@redhat.com>
Co-authored-by: Alexander Schwartz <aschwart@redhat.com>

.github/workflows/ci.yml

@@ -830,7 +830,7 @@ jobs:
 
       - name: Run cluster tests with mtls
         run: |
-          ./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-cluster-quarkus,db-postgres "-Dwebdriver.chrome.driver=$CHROMEWEBDRIVER/chromedriver" -Dsession.cache.owners=2 -Dtest=RealmInvalidationClusterTest -Dauth.server.jgroups.mtls=true -pl testsuite/integration-arquillian/tests/base
+          ./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-cluster-quarkus,db-postgres "-Dwebdriver.chrome.driver=$CHROMEWEBDRIVER/chromedriver" -Dsession.cache.owners=2 -Dtest=RealmInvalidationClusterTest,JGroupsCertificateRotationClusterTest -Dauth.server.jgroups.mtls=true -Dauth.server.quarkus.log-level=org.keycloak.infinispan.module.certificates:DEBUG -pl testsuite/integration-arquillian/tests/base
 
       - name: Upload JVM Heapdumps
         if: always()

docs/guides/server/caching.adoc

@@ -314,6 +314,9 @@ The generated certificate and associated keys are stored within the database of
 
 To enable zero-configuration TLS encryption, set the `cache-embedded-mtls-enabled` option to true.
 No other `cache-embedded-mtls-*` must be set to enable the zero-configuration mode.
+
+The `cache-embedded-mtls-rotation-interval-days` option (default: 30 days) configures the certificate rotation period, and the certificate's expiration duration is calculated as twice the specified interval.
+
 ====
 
 For JGroups stacks with `UDP` or `TCP_NIO2`, see the http://jgroups.org/manual5/index.html#ENCRYPT[JGroups Encryption documentation] on how to set up the protocol stack.

model/infinispan/pom.xml

@@ -83,6 +83,11 @@
             <artifactId>infinispan-component-annotations</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.infinispan</groupId>
+            <artifactId>infinispan-component-processor</artifactId>
+            <scope>provided</scope>
+        </dependency>
 
         <dependency>
             <groupId>junit</groupId>

model/infinispan/src/main/java/org/keycloak/infinispan/module/KeycloakModule.java

@@ -0,0 +1,20 @@
+package org.keycloak.infinispan.module;
+
+import org.infinispan.configuration.global.GlobalConfiguration;
+import org.infinispan.factories.GlobalComponentRegistry;
+import org.infinispan.factories.annotations.InfinispanModule;
+import org.infinispan.factories.impl.BasicComponentRegistry;
+import org.infinispan.lifecycle.ModuleLifecycle;
+import org.keycloak.infinispan.module.certificates.CertificateReloadManager;
+
+@InfinispanModule(name = "keycloak", requiredModules = {"core"})
+public class KeycloakModule implements ModuleLifecycle {
+
+    @Override
+    public void cacheManagerStarting(GlobalComponentRegistry gcr, GlobalConfiguration globalConfiguration) {
+        // start certificate reload manager before the JGroupsTransport
+        gcr.getComponent(BasicComponentRegistry.class)
+                .getComponent(CertificateReloadManager.class)
+                .running();
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/certificates/CertificateReloadManager.java

@@ -0,0 +1,283 @@
+package org.keycloak.infinispan.module.certificates;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.math.BigInteger;
+import java.security.GeneralSecurityException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.infinispan.commons.api.Lifecycle;
+import org.infinispan.factories.KnownComponentNames;
+import org.infinispan.factories.annotations.ComponentName;
+import org.infinispan.factories.annotations.Inject;
+import org.infinispan.factories.annotations.Start;
+import org.infinispan.factories.annotations.Stop;
+import org.infinispan.factories.scopes.Scope;
+import org.infinispan.factories.scopes.Scopes;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.notifications.Listener;
+import org.infinispan.notifications.cachemanagerlistener.CacheManagerNotifier;
+import org.infinispan.notifications.cachemanagerlistener.annotation.Merged;
+import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
+import org.infinispan.notifications.cachemanagerlistener.event.ViewChangedEvent;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.util.concurrent.BlockingManager;
+import org.jboss.logging.Logger;
+import org.keycloak.common.util.CertificateUtils;
+import org.keycloak.common.util.KeyUtils;
+import org.keycloak.common.util.Time;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.storage.configuration.ServerConfigStorageProvider;
+
+import static org.keycloak.infinispan.module.certificates.JGroupsCertificate.toJson;
+
+/**
+ * Class to handle JGroups certificate reloading for encryption (mTLS).
+ * <p>
+ * This class is attached to Infinispan lifecycle, and it starts/stops together with the {@link EmbeddedCacheManager}.
+ * <p>
+ * It provides two public methods, {@link #rotateCertificate()} to force a certificate rotation without waiting for the
+ * configured period, and {@link #reloadCertificate()} to force a certificate reloading from storage and schedule the
+ * next rotation.
+ * <p>
+ * When the timer expires, only the cluster coordinator generates a new certificate. It notifies the other cluster
+ * members that a new certificate is available in storage. Both the key and trust stores keep a hold of the old and the
+ * new certificates.
+ * <p>
+ * Last, but not least, it listens to topology changes and, if the coordinator crashes, the new re-elected coordinator
+ * will continue to perform its duties to rotate the certificate.
+ */
+@Scope(Scopes.GLOBAL)
+@Listener
+public class CertificateReloadManager implements Lifecycle {
+
+    private static final Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
+    public static final String CERTIFICATE_ID = "crt_jgroups";
+    private static final String JGROUPS_SUBJECT = "jgroups";
+    private static final Duration RETRY_WAIT_TIME = Duration.ofMinutes(1);
+
+    private final KeycloakSessionFactory sessionFactory;
+    private final JGroupsCertificateHolder certificateHolder;
+    private volatile long rotationSeconds;
+    private final AutoCloseableLock lock;
+    private ScheduledFuture<?> scheduledFuture;
+
+    @Inject
+    EmbeddedCacheManager cacheManager;
+
+    @Inject
+    CacheManagerNotifier notifier;
+
+    @ComponentName(KnownComponentNames.EXPIRATION_SCHEDULED_EXECUTOR)
+    @Inject
+    ScheduledExecutorService scheduledExecutorService;
+
+    @Inject
+    BlockingManager blockingManager;
+
+    public CertificateReloadManager(KeycloakSessionFactory sessionFactory, JGroupsCertificateHolder certificateHolder, int rotationDays) {
+        this.sessionFactory = sessionFactory;
+        this.certificateHolder = certificateHolder;
+        this.rotationSeconds = TimeUnit.DAYS.toSeconds(rotationDays);
+        lock = new AutoCloseableLock(new ReentrantLock());
+    }
+
+    @Override
+    @Start
+    public void start() {
+        logger.debug("Starting JGroups certificate reload manager");
+        notifier.addListener(this);
+        reloadCertificate();
+        certificateHolder.setExceptionHandler(this::onInvalidCertificate);
+    }
+
+    @Override
+    @Stop
+    public void stop() {
+        logger.debug("Stopping JGroups certificate reload manager");
+        notifier.removeListener(this);
+        lock.lock();
+        try (lock) {
+            if (scheduledFuture == null) {
+                return;
+            }
+            scheduledFuture.cancel(true);
+        }
+    }
+
+    /**
+     * Creates and reload a new certificate.
+     */
+    public void rotateCertificate() {
+        logger.debug("Rotate JGroups certificate");
+        lock.lock();
+        try (lock) {
+            KeycloakModelUtils.runJobInTransaction(sessionFactory, this::replaceCertificateInTransaction);
+            sendReloadNotification();
+        } catch (RuntimeException e) {
+            logger.warn("Failed to rotate JGroups certificate", e);
+            retry(this::rotateCertificate, "retry-rotate");
+        }
+    }
+
+    /**
+     * Reloads the certificate from storage.
+     */
+    public void reloadCertificate() {
+        logger.debug("Reload JGroups Certificate");
+        lock.lock();
+        try (lock) {
+            var maybeCrt = KeycloakModelUtils.runJobInTransactionWithResult(sessionFactory, CertificateReloadManager::loadCertificateInTransaction);
+            if (maybeCrt.isEmpty()) {
+                return;
+            }
+            var crt = JGroupsCertificate.fromJson(maybeCrt.get());
+            certificateHolder.useCertificate(crt);
+        } catch (GeneralSecurityException | IOException e) {
+            logger.warn("Failed to reload JGroups certificate", e);
+            retry(this::reloadCertificate, "retry-reload");
+        } finally {
+            scheduleNextRotation();
+        }
+    }
+
+    @ViewChanged
+    @Merged
+    public void onViewChanged(ViewChangedEvent event) {
+        logger.debug("On view changed");
+        // probably a waste to reload, but if we have a partition, we reload the most recent certificate stored.
+        reloadCertificate();
+    }
+
+    // testing purpose
+    public JGroupsCertificate currentCertificate() {
+        return certificateHolder.getCertificateInUse();
+    }
+
+    // testing purpose
+    public void setRotationSeconds(long seconds) {
+        this.rotationSeconds = seconds;
+    }
+
+    // testing purpose
+    public boolean isCoordinator() {
+        return cacheManager.isCoordinator();
+    }
+
+    // testing purpose
+    public boolean hasRotationTask() {
+        lock.lock();
+        try (lock) {
+            return scheduledFuture != null;
+        }
+    }
+
+    private void onInvalidCertificate() {
+        logger.debug("On certificate exception");
+        blockingManager.runBlocking(this::reloadCertificate, "invalid-certificate");
+    }
+
+    private void onCertificateReloadResponse(Address address, Void unused, Throwable throwable) {
+        if (throwable != null) {
+            logger.warnf(throwable, "Node %s failed to handle JGroups certificate reload notification.", address);
+            retry(() -> sendReloadNotification(address), "retry-notification");
+        }
+    }
+
+    private void scheduleNextRotation() {
+        lock.lock();
+        try (lock) {
+            if (scheduledFuture != null) {
+                scheduledFuture.cancel(false);
+            }
+            if (!isCoordinator()) {
+                return;
+            }
+            var crt = certificateHolder.getCertificateInUse();
+            var delay = delayUntilNextRotation(crt.getCertificate().getNotBefore().toInstant(), crt.getCertificate().getNotAfter().toInstant());
+            logger.debugf("Next rotation in %s", delay);
+            if (delay.isZero()) {
+                blockingManager.runBlocking(this::rotateCertificate, "rotate");
+                return;
+            }
+            scheduledFuture = scheduledExecutorService.schedule(() -> blockingManager.runBlocking(this::rotateCertificate, "rotate"), delay.toSeconds(), TimeUnit.SECONDS);
+        }
+    }
+
+    private void replaceCertificateInTransaction(KeycloakSession session) {
+        var storage = session.getProvider(ServerConfigStorageProvider.class);
+        var holder = certificateHolder.getCertificateInUse();
+        storage.replace(CERTIFICATE_ID, holder::isSameAlias, () -> generateSelfSignedCertificate(rotationSeconds * 2L));
+    }
+
+    private static Optional<String> loadCertificateInTransaction(KeycloakSession session) {
+        return session.getProvider(ServerConfigStorageProvider.class).find(CERTIFICATE_ID);
+    }
+
+    private Duration delayUntilNextRotation(Instant certificateStartInstant, Instant certificateEndInstant) {
+        var rotationInstant = certificateStartInstant.plus(Duration.ofSeconds(rotationSeconds));
+
+        // Avoid the current certificate to expire if the old duration was shorter than the new duration
+        var rotationInstantOldCertificate = certificateStartInstant.plus(Duration.between(certificateStartInstant, certificateEndInstant).dividedBy(2));
+        if (rotationInstantOldCertificate.isBefore(rotationInstant)) {
+            rotationInstant = rotationInstantOldCertificate;
+        }
+
+        var secondsLeft = Instant.ofEpochSecond(Time.currentTime()).until(rotationInstant, ChronoUnit.SECONDS);
+        return secondsLeft > 0 ? Duration.ofSeconds(secondsLeft) : Duration.ZERO;
+    }
+
+    private void sendReloadNotification() {
+        cacheManager.executor()
+                .allNodeSubmission()
+                .submitConsumer(ReloadCertificateFunction.getInstance(), this::onCertificateReloadResponse);
+    }
+
+    private void sendReloadNotification(Address destination) {
+        cacheManager.executor()
+                .filterTargets(destination::equals)
+                .submitConsumer(ReloadCertificateFunction.getInstance(), this::onCertificateReloadResponse);
+    }
+
+    private void retry(Runnable runnable, String traceId) {
+        scheduledExecutorService.schedule(() -> blockingManager.runBlocking(runnable, traceId), RETRY_WAIT_TIME.toSeconds(), TimeUnit.SECONDS);
+    }
+
+    public static String generateSelfSignedCertificate(long validForSeconds) {
+        var endDate = Date.from(Instant.now().plus(validForSeconds, ChronoUnit.SECONDS));
+        var keyPair = KeyUtils.generateRsaKeyPair(2048);
+        var certificate = CertificateUtils.generateV1SelfSignedCertificate(keyPair, JGROUPS_SUBJECT, BigInteger.valueOf(System.currentTimeMillis()), endDate);
+
+        logger.debugf("Created JGroups certificate. Valid until %s", certificate.getNotAfter());
+
+        var entity = new JGroupsCertificate();
+        entity.setCertificate(certificate);
+        entity.setKeyPair(keyPair);
+        entity.setAlias(UUID.randomUUID().toString());
+        return toJson(entity);
+    }
+
+    private record AutoCloseableLock(ReentrantLock innerLock) implements AutoCloseable {
+
+        public void lock() {
+            innerLock.lock();
+        }
+
+        @Override
+        public void close() {
+            innerLock.unlock();
+        }
+    }
+
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/certificates/JGroupsCertificate.java

@@ -15,21 +15,24 @@
  * limitations under the License.
  */
 
-package org.keycloak.quarkus.runtime.storage.infinispan.jgroups.impl;
+package org.keycloak.infinispan.module.certificates;
 
 import java.security.KeyPair;
+import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import java.util.Objects;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import org.keycloak.common.util.PemUtils;
+import org.keycloak.util.JsonSerialization;
 
 /**
  * JPA entity to store the {@link X509Certificate} and {@link KeyPair}.
  */
 @SuppressWarnings("unused")
-public class CertificateEntity {
+public class JGroupsCertificate {
 
     @JsonProperty("prvKey")
     private String privateKeyPem;
@@ -40,13 +43,10 @@ public class CertificateEntity {
     @JsonProperty("crt")
     private String certificatePem;
 
-    public CertificateEntity() {
-    }
+    @JsonProperty("alias")
+    private String alias;
 
-    public CertificateEntity(String privateKeyPem, String publicKeyPem, String certificatePem) {
-        this.privateKeyPem = Objects.requireNonNull(privateKeyPem);
-        this.publicKeyPem = Objects.requireNonNull(publicKeyPem);
-        this.certificatePem = Objects.requireNonNull(certificatePem);
+    public JGroupsCertificate() {
     }
 
     public String getCertificatePem() {
@@ -73,6 +73,14 @@ public class CertificateEntity {
         this.publicKeyPem = publicKeyPem;
     }
 
+    public String getAlias() {
+        return alias;
+    }
+
+    public void setAlias(String alias) {
+        this.alias = alias;
+    }
+
     @JsonIgnore
     public void setCertificate(X509Certificate certificate) {
         Objects.requireNonNull(certificate);
@@ -98,11 +106,16 @@ public class CertificateEntity {
         return new KeyPair(pub, prv);
     }
 
+    @JsonIgnore
+    public PrivateKey getPrivateKey() {
+        return PemUtils.decodePrivateKey(getPrivateKeyPem());
+    }
+
     @Override
     public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
 
-        CertificateEntity that = (CertificateEntity) o;
+        JGroupsCertificate that = (JGroupsCertificate) o;
         return Objects.equals(privateKeyPem, that.privateKeyPem) &&
                 Objects.equals(publicKeyPem, that.publicKeyPem) &&
                 Objects.equals(certificatePem, that.certificatePem);
@@ -115,4 +128,24 @@ public class CertificateEntity {
         result = 31 * result + Objects.hashCode(certificatePem);
         return result;
     }
+
+    public boolean isSameAlias(String jsonCertificate) {
+        return Objects.equals(alias, fromJson(jsonCertificate).getAlias());
+    }
+
+    public static String toJson(JGroupsCertificate entity) {
+        try {
+            return JsonSerialization.mapper.writeValueAsString(entity);
+        } catch (JsonProcessingException e) {
+            throw new IllegalStateException("Should never happen!", e);
+        }
+    }
+
+    public static JGroupsCertificate fromJson(String json) {
+        try {
+            return JsonSerialization.mapper.readValue(json, JGroupsCertificate.class);
+        } catch (JsonProcessingException e) {
+            throw new IllegalStateException("Should never happen!", e);
+        }
+    }
 }

model/infinispan/src/main/java/org/keycloak/infinispan/module/certificates/JGroupsCertificateHolder.java

@@ -0,0 +1,120 @@
+package org.keycloak.infinispan.module.certificates;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.util.Objects;
+
+import org.jboss.logging.Logger;
+import org.keycloak.common.crypto.CryptoIntegration;
+import org.keycloak.common.util.KeystoreUtil;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509ExtendedKeyManager;
+import javax.net.ssl.X509ExtendedTrustManager;
+
+//TODO move to SPI https://github.com/keycloak/keycloak/issues/37325
+
+/**
+ * Holds the JGroups certificate and updates the {@link X509ExtendedKeyManager} and {@link X509ExtendedTrustManager}
+ * used by the TLS/SSL sockets.
+ */
+public class JGroupsCertificateHolder {
+
+    private static final Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
+    private static final char[] KEY_PASSWORD = "jgroups-password".toCharArray();
+
+    private volatile JGroupsCertificate certificate;
+    private final ReloadingX509ExtendedKeyManager keyManager;
+    private final ReloadingX509ExtendedTrustManager trustManager;
+
+
+    private JGroupsCertificateHolder(ReloadingX509ExtendedKeyManager keyManager, ReloadingX509ExtendedTrustManager trustManager, JGroupsCertificate certificate) {
+        this.keyManager = keyManager;
+        this.trustManager = trustManager;
+        this.certificate = certificate;
+    }
+
+    public static JGroupsCertificateHolder create(JGroupsCertificate certificate) throws GeneralSecurityException, IOException {
+        Objects.requireNonNull(certificate);
+        var km = createKeyManager(certificate);
+        var tm = createTrustManager(null, certificate);
+        var crt =certificate.getCertificate();
+        logger.debugf("Using JGroups certificate (serial: %s). Valid until %s", crt.getSerialNumber(), crt.getNotAfter());
+        return new JGroupsCertificateHolder(new ReloadingX509ExtendedKeyManager(km), new ReloadingX509ExtendedTrustManager(tm), certificate);
+    }
+
+    public JGroupsCertificate getCertificateInUse() {
+        return certificate;
+    }
+
+    public void useCertificate(JGroupsCertificate certificate) throws GeneralSecurityException, IOException {
+        Objects.requireNonNull(certificate);
+        if (Objects.equals(certificate.getAlias(), this.certificate.getAlias())) {
+            return;
+        }
+        var crt =certificate.getCertificate();
+        logger.debugf("Using JGroups certificate (serial: %s). Valid until %s", crt.getSerialNumber(), crt.getNotAfter());
+        if (this.certificate != null) {
+            crt = this.certificate.getCertificate();
+            logger.debugf("Old JGroups certificate (serial: %s). Valid until %s", crt.getSerialNumber(), crt.getNotAfter());
+        }
+        var km = createKeyManager(certificate);
+        var tm = createTrustManager(this.certificate, certificate);
+        keyManager.reload(km);
+        trustManager.reload(tm);
+        this.certificate = certificate;
+    }
+
+    public X509ExtendedKeyManager keyManager() {
+        return keyManager;
+    }
+
+    public X509ExtendedTrustManager trustManager() {
+        return trustManager;
+    }
+
+    public void setExceptionHandler(Runnable runnable) {
+        trustManager.setExceptionHandler(runnable);
+    }
+
+    private static X509ExtendedKeyManager createKeyManager(JGroupsCertificate newCertificate) throws GeneralSecurityException, IOException {
+        var ks = CryptoIntegration.getProvider().getKeyStore(KeystoreUtil.KeystoreFormat.JKS);
+        ks.load(null, null);
+        ks.setKeyEntry(newCertificate.getAlias(), newCertificate.getPrivateKey(), KEY_PASSWORD, new java.security.cert.Certificate[]{newCertificate.getCertificate()});
+        var kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        kmf.init(ks, KEY_PASSWORD);
+        for (KeyManager km : kmf.getKeyManagers()) {
+            if (km instanceof X509ExtendedKeyManager) {
+                return (X509ExtendedKeyManager) km;
+            }
+        }
+        throw new GeneralSecurityException("Could not obtain an X509ExtendedKeyManager");
+    }
+
+    private static X509ExtendedTrustManager createTrustManager(JGroupsCertificate oldCertificate, JGroupsCertificate newCertificate) throws GeneralSecurityException, IOException {
+        var ks = CryptoIntegration.getProvider().getKeyStore(KeystoreUtil.KeystoreFormat.JKS);
+        ks.load(null, null);
+        if (oldCertificate != null) {
+            addCertificateEntry(ks, oldCertificate);
+        }
+        addCertificateEntry(ks, newCertificate);
+        var tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        tmf.init(ks);
+        for (TrustManager tm : tmf.getTrustManagers()) {
+            if (tm instanceof X509ExtendedTrustManager) {
+                return (X509ExtendedTrustManager) tm;
+            }
+        }
+        throw new GeneralSecurityException("Could not obtain an X509TrustManager");
+    }
+
+    private static void addCertificateEntry(KeyStore store, JGroupsCertificate certificate) throws KeyStoreException {
+        store.setCertificateEntry(certificate.getAlias(), certificate.getCertificate());
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/certificates/ReloadCertificateFunction.java

@@ -0,0 +1,34 @@
+package org.keycloak.infinispan.module.certificates;
+
+import java.util.function.Function;
+
+import org.infinispan.factories.GlobalComponentRegistry;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.protostream.annotations.ProtoFactory;
+import org.infinispan.protostream.annotations.ProtoTypeId;
+import org.keycloak.marshalling.Marshalling;
+
+/**
+ * Reloads the JGroups certificate
+ */
+@ProtoTypeId(Marshalling.RELOAD_CERTIFICATE_FUNCTION)
+public final class ReloadCertificateFunction implements Function<EmbeddedCacheManager, Void> {
+
+    private static final ReloadCertificateFunction INSTANCE = new ReloadCertificateFunction();
+
+    private ReloadCertificateFunction() {}
+
+    @ProtoFactory
+    public static ReloadCertificateFunction getInstance() {
+        return INSTANCE;
+    }
+
+    @Override
+    public Void apply(EmbeddedCacheManager embeddedCacheManager) {
+        var crm = GlobalComponentRegistry.componentOf(embeddedCacheManager, CertificateReloadManager.class);
+        if (crm != null) {
+            crm.reloadCertificate();
+        }
+        return null;
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/certificates/ReloadingX509ExtendedKeyManager.java

@@ -0,0 +1,77 @@
+package org.keycloak.infinispan.module.certificates;
+
+import java.lang.invoke.MethodHandles;
+import java.net.Socket;
+import java.security.Principal;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.jboss.logging.Logger;
+
+import javax.net.ssl.X509ExtendedKeyManager;
+
+/**
+ * A {@link X509ExtendedKeyManager} implementation that allows to update the keys and certificates at runtime.
+ */
+class ReloadingX509ExtendedKeyManager extends X509ExtendedKeyManager {
+
+    private static final Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
+    private volatile X509ExtendedKeyManager delegate;
+
+    public ReloadingX509ExtendedKeyManager(X509ExtendedKeyManager delegate) {
+        this.delegate = Objects.requireNonNull(delegate);
+    }
+
+    @Override
+    public String[] getClientAliases(String keyType, Principal[] issuers) {
+        var r = delegate.getClientAliases(keyType, issuers);
+        if (logger.isDebugEnabled() && r != null) {
+            logger.debugf("getClientAliases - %s", Arrays.toString(r));
+        }
+        return r;
+    }
+
+    @Override
+    public String chooseClientAlias(String[] keyType, Principal[] issuers, Socket socket) {
+        var r = delegate.chooseClientAlias(keyType, issuers, socket);
+        logger.debugf("chooseClientAlias - %s", r);
+        return r;
+    }
+
+    @Override
+    public String[] getServerAliases(String keyType, Principal[] issuers) {
+        var r = delegate.getServerAliases(keyType, issuers);
+        if (logger.isDebugEnabled()) {
+            logger.debugf("getServerAliases - %s", Arrays.toString(r));
+        }
+        return r;
+    }
+
+    @Override
+    public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) {
+        var r = delegate.chooseServerAlias(keyType, issuers, socket);
+        logger.debugf("chooseServerAlias - %s", r);
+        return r;
+    }
+
+    @Override
+    public X509Certificate[] getCertificateChain(String alias) {
+        var r = delegate.getCertificateChain(alias);
+        if (logger.isDebugEnabled() && r != null) {
+            logger.debugf("getCertificateChain - serial numbers: %s", Arrays.stream(r).map(X509Certificate::getSerialNumber).map(String::valueOf).collect(Collectors.joining(", ")));
+        }
+        return r;
+    }
+
+    @Override
+    public PrivateKey getPrivateKey(String alias) {
+        return delegate.getPrivateKey(alias);
+    }
+
+    public void reload(X509ExtendedKeyManager keyManager) {
+        delegate = Objects.requireNonNull(keyManager);
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/certificates/ReloadingX509ExtendedTrustManager.java

@@ -0,0 +1,101 @@
+package org.keycloak.infinispan.module.certificates;
+
+import java.net.Socket;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Objects;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedTrustManager;
+
+/**
+ * A {@link X509ExtendedTrustManager} that allows to update the trusted certificate at runtime.
+ */
+class ReloadingX509ExtendedTrustManager extends X509ExtendedTrustManager {
+
+    private volatile X509ExtendedTrustManager delegate;
+    private volatile Runnable onException;
+
+    public ReloadingX509ExtendedTrustManager(X509ExtendedTrustManager delegate) {
+        this.delegate = Objects.requireNonNull(delegate);
+        this.onException = () -> {
+        };
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException {
+        try {
+            delegate.checkClientTrusted(chain, authType, engine);
+        } catch (CertificateException e) {
+            onException.run();
+            throw e;
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
+        try {
+            delegate.checkClientTrusted(chain, authType, socket);
+        } catch (CertificateException e) {
+            onException.run();
+            throw e;
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException {
+        try {
+
+            delegate.checkServerTrusted(chain, authType, engine);
+        } catch (CertificateException e) {
+            onException.run();
+            throw e;
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
+        try {
+
+            delegate.checkServerTrusted(chain, authType, socket);
+        } catch (CertificateException e) {
+            onException.run();
+            throw e;
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+        try {
+
+            delegate.checkClientTrusted(chain, authType);
+        } catch (CertificateException e) {
+            onException.run();
+            throw e;
+        }
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+        try {
+            delegate.checkServerTrusted(chain, authType);
+        } catch (CertificateException e) {
+            onException.run();
+            throw e;
+        }
+    }
+
+    @Override
+    public X509Certificate[] getAcceptedIssuers() {
+        return delegate.getAcceptedIssuers();
+    }
+
+    public void setExceptionHandler(Runnable runnable) {
+        this.onException = Objects.requireNonNullElse(runnable, () -> {
+        });
+    }
+
+    public void reload(X509ExtendedTrustManager trustManager) {
+        delegate = Objects.requireNonNull(trustManager);
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/configuration/global/KeycloakConfiguration.java

@@ -0,0 +1,53 @@
+package org.keycloak.infinispan.module.configuration.global;
+
+import org.infinispan.commons.configuration.BuiltBy;
+import org.infinispan.commons.configuration.attributes.AttributeDefinition;
+import org.infinispan.commons.configuration.attributes.AttributeSet;
+import org.keycloak.infinispan.module.certificates.JGroupsCertificateHolder;
+import org.keycloak.models.KeycloakSessionFactory;
+
+@BuiltBy(KeycloakConfigurationBuilder.class)
+public class KeycloakConfiguration {
+
+    static final AttributeDefinition<KeycloakSessionFactory> KEYCLOAK_SESSION_FACTORY = AttributeDefinition.builder("keycloak-session-factory", null, KeycloakSessionFactory.class)
+            .global(true)
+            .autoPersist(false)
+            .immutable()
+            .build();
+    static final AttributeDefinition<JGroupsCertificateHolder> JGROUPS_CERTIFICATE_HOLDER = AttributeDefinition.builder("jgroups-certificate-holder", null, JGroupsCertificateHolder.class)
+            .global(true)
+            .autoPersist(false)
+            .immutable()
+            .build();
+    static final AttributeDefinition<Integer> JGROUPS_CERTIFICATE_ROTATION = AttributeDefinition.builder("jgroups-certificate-rotation", 30, Integer.class)
+            .global(true)
+            .autoPersist(false)
+            .immutable()
+            .build();
+
+    private final AttributeSet attributes;
+
+    static AttributeSet attributeSet() {
+        return new AttributeSet(KeycloakConfiguration.class, KEYCLOAK_SESSION_FACTORY, JGROUPS_CERTIFICATE_HOLDER, JGROUPS_CERTIFICATE_ROTATION);
+    }
+
+    KeycloakConfiguration(AttributeSet attributes) {
+        this.attributes = attributes;
+    }
+
+    AttributeSet attributes() {
+        return attributes;
+    }
+
+    public KeycloakSessionFactory keycloakSessionFactory() {
+        return attributes.attribute(KEYCLOAK_SESSION_FACTORY).get();
+    }
+
+    public JGroupsCertificateHolder jGroupsCertificateHolder() {
+        return attributes.attribute(JGROUPS_CERTIFICATE_HOLDER).get();
+    }
+
+    public int jgroupsCertificateRotation() {
+        return attributes.attribute(JGROUPS_CERTIFICATE_ROTATION).get();
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/configuration/global/KeycloakConfigurationBuilder.java

@@ -0,0 +1,53 @@
+package org.keycloak.infinispan.module.configuration.global;
+
+import org.infinispan.commons.configuration.Builder;
+import org.infinispan.commons.configuration.Combine;
+import org.infinispan.commons.configuration.attributes.AttributeSet;
+import org.infinispan.configuration.global.GlobalConfigurationBuilder;
+import org.keycloak.infinispan.module.certificates.JGroupsCertificateHolder;
+import org.keycloak.models.KeycloakSessionFactory;
+
+public class KeycloakConfigurationBuilder implements Builder<KeycloakConfiguration> {
+
+    private final AttributeSet attributes;
+
+    public KeycloakConfigurationBuilder(GlobalConfigurationBuilder unused) {
+        attributes = KeycloakConfiguration.attributeSet();
+    }
+
+    @Override
+    public KeycloakConfiguration create() {
+        return new KeycloakConfiguration(attributes.protect());
+    }
+
+    @Override
+    public Builder<?> read(KeycloakConfiguration template, Combine combine) {
+        attributes.read(template.attributes(), combine);
+        return this;
+    }
+
+    @Override
+    public AttributeSet attributes() {
+        return attributes;
+    }
+
+    @Override
+    public void validate() {
+
+    }
+
+    public KeycloakConfigurationBuilder setKeycloakSessionFactory(KeycloakSessionFactory keycloakSessionFactory) {
+        attributes.attribute(KeycloakConfiguration.KEYCLOAK_SESSION_FACTORY).set(keycloakSessionFactory);
+        return this;
+    }
+
+    public KeycloakConfigurationBuilder setJGroupCertificateHolder(JGroupsCertificateHolder jGroupsCertificateHolder) {
+        attributes.attribute(KeycloakConfiguration.JGROUPS_CERTIFICATE_HOLDER).set(jGroupsCertificateHolder);
+        return this;
+    }
+
+    public KeycloakConfigurationBuilder setJGroupsCertificateRotation(int days) {
+        attributes.attribute(KeycloakConfiguration.JGROUPS_CERTIFICATE_ROTATION).set(days);
+        return this;
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/module/factory/CertificateReloadManagerFactory.java

@@ -0,0 +1,25 @@
+package org.keycloak.infinispan.module.factory;
+
+import org.infinispan.factories.AbstractComponentFactory;
+import org.infinispan.factories.AutoInstantiableFactory;
+import org.infinispan.factories.annotations.DefaultFactoryFor;
+import org.keycloak.infinispan.module.certificates.CertificateReloadManager;
+import org.keycloak.infinispan.module.configuration.global.KeycloakConfiguration;
+
+@DefaultFactoryFor(classes = CertificateReloadManager.class)
+public class CertificateReloadManagerFactory extends AbstractComponentFactory implements AutoInstantiableFactory {
+
+    @Override
+    public Object construct(String componentName) {
+        var kcConfig = globalConfiguration.module(KeycloakConfiguration.class);
+        if (kcConfig == null) {
+            return null;
+        }
+        var sessionFactory = kcConfig.keycloakSessionFactory();
+        var certificateHolder = kcConfig.jGroupsCertificateHolder();
+        if (sessionFactory == null || certificateHolder == null) {
+            throw new IllegalStateException("KeycloakConfiguration is not null when the certificate reload is required.");
+        }
+        return new CertificateReloadManager(sessionFactory, certificateHolder, kcConfig.jgroupsCertificateRotation());
+    }
+}

model/infinispan/src/main/java/org/keycloak/marshalling/KeycloakModelSchema.java

@@ -33,6 +33,7 @@ import org.keycloak.cluster.infinispan.LockEntry;
 import org.keycloak.cluster.infinispan.LockEntryPredicate;
 import org.keycloak.cluster.infinispan.WrapperClusterEvent;
 import org.keycloak.component.ComponentModel;
+import org.keycloak.infinispan.module.certificates.ReloadCertificateFunction;
 import org.keycloak.keys.infinispan.PublicKeyStorageInvalidationEvent;
 import org.keycloak.models.UserSessionModel;
 import org.keycloak.models.cache.infinispan.ClearCacheEvent;
@@ -101,8 +102,8 @@ import org.keycloak.models.sessions.infinispan.stream.CollectionToStreamMapper;
 import org.keycloak.models.sessions.infinispan.stream.GroupAndCountCollectorSupplier;
 import org.keycloak.models.sessions.infinispan.stream.MapEntryToKeyMapper;
 import org.keycloak.models.sessions.infinispan.stream.SessionPredicate;
-import org.keycloak.models.sessions.infinispan.stream.SessionWrapperPredicate;
 import org.keycloak.models.sessions.infinispan.stream.SessionUnwrapMapper;
+import org.keycloak.models.sessions.infinispan.stream.SessionWrapperPredicate;
 import org.keycloak.models.sessions.infinispan.stream.UserSessionPredicate;
 import org.keycloak.sessions.CommonClientSessionModel;
 import org.keycloak.storage.UserStorageProviderModel;
@@ -206,7 +207,6 @@ import org.keycloak.storage.managers.UserStorageSyncManager;
                 UserFullInvalidationEvent.class,
                 UserUpdatedEvent.class,
 
-
                 // sessions.infinispan.entities package
                 AuthenticatedClientSessionStore.class,
                 AuthenticatedClientSessionEntity.class,
@@ -227,6 +227,9 @@ import org.keycloak.storage.managers.UserStorageSyncManager;
                 GroupAndCountCollectorSupplier.class,
                 MapEntryToKeyMapper.class,
                 SessionUnwrapMapper.class,
+
+                // infinispan.module.certificates
+                ReloadCertificateFunction.class,
         }
 )
 public interface KeycloakModelSchema extends GeneratedSchema {

model/infinispan/src/main/java/org/keycloak/marshalling/Marshalling.java

@@ -163,6 +163,8 @@ public final class Marshalling {
     public static final int PERMISSION_TICKET_REMOVED_EVENT = 65613;
     public static final int PERMISSION_TICKET_UPDATED_EVENT = 65614;
 
+    public static final int RELOAD_CERTIFICATE_FUNCTION = 65615;
+
     public static void configure(GlobalConfigurationBuilder builder) {
         builder.serialization()
                 .addContextInitializer(KeycloakModelSchema.INSTANCE);

model/jpa/src/main/java/org/keycloak/storage/configuration/jpa/JpaServerConfigStorageProvider.java

@@ -19,6 +19,7 @@ package org.keycloak.storage.configuration.jpa;
 
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import jakarta.persistence.EntityManager;
@@ -67,7 +68,7 @@ public class JpaServerConfigStorageProvider implements ServerConfigStorageProvid
 
     @Override
     public String loadOrCreate(String key, Supplier<String> valueGenerator) {
-        var entity = getEntity(key, LockModeType.WRITE);
+        var entity = getEntity(key, LockModeType.OPTIMISTIC);
         if (entity != null) {
             return entity.getValue();
         }
@@ -79,6 +80,19 @@ public class JpaServerConfigStorageProvider implements ServerConfigStorageProvid
         return value;
     }
 
+    @Override
+    public boolean replace(String key, Predicate<String> replacePredicate, Supplier<String> valueGenerator) {
+        Objects.requireNonNull(replacePredicate);
+        Objects.requireNonNull(valueGenerator);
+        var entity = getEntity(key, LockModeType.OPTIMISTIC);
+        if (entity == null || !replacePredicate.test(entity.getValue())) {
+            return false;
+        }
+        entity.setValue(valueGenerator.get());
+        entityManager.merge(entity);
+        return true;
+    }
+
     @Override
     public void close() {
         //no-op

model/storage/src/main/java/org/keycloak/storage/configuration/ServerConfigStorageProvider.java

@@ -17,7 +17,9 @@
 
 package org.keycloak.storage.configuration;
 
+import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import org.keycloak.provider.Provider;
@@ -69,4 +71,32 @@ public interface ServerConfigStorageProvider extends Provider {
      */
     String loadOrCreate(String key, Supplier<String> valueGenerator);
 
+    /**
+     * Same as {@code loadOrCreate(key, () -> value)}.
+     *
+     * @see #loadOrCreate(String, Supplier)
+     */
+    default String loadOrCreate(String key, String value) {
+        return loadOrCreate(key, () -> value);
+    }
+
+    /**
+     * Same as {@code replace(key, Objects.requireNonNull(expected)::equals, () -> Objects.requireNonNull(newValue))}.
+     *
+     * @see #replace(String, Predicate, Supplier)
+     */
+    default boolean replace(String key, String expected, String newValue) {
+        return replace(key, Objects.requireNonNull(expected)::equals, () -> Objects.requireNonNull(newValue));
+    }
+
+    /**
+     * Replaces the value specified by {@code key} if the {@link Predicate} return a {@code true} value.
+     *
+     * @param key              The {@code key} whose associated value is to be replaced.
+     * @param replacePredicate The {@link Predicate} to signal if the value should be replaced.
+     * @param valueGenerator   The {@link Supplier} to generate the value if it is should be replaced.
+     * @return {@code true} if the value is replaced, and {@code false} otherwise.
+     */
+    boolean replace(String key, Predicate<String> replacePredicate, Supplier<String> valueGenerator);
+
 }

quarkus/config-api/src/main/java/org/keycloak/config/CachingOptions.java

@@ -15,6 +15,7 @@ public class CachingOptions {
     public static final String CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD_PROPERTY = CACHE_EMBEDDED_MTLS_PREFIX + "-key-store-password";
     public static final String CACHE_EMBEDDED_MTLS_TRUSTSTORE_FILE_PROPERTY = CACHE_EMBEDDED_MTLS_PREFIX + "-trust-store-file";
     public static final String CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD_PROPERTY = CACHE_EMBEDDED_MTLS_PREFIX + "-trust-store-password";
+    public static final String CACHE_EMBEDDED_MTLS_ROTATION_PROPERTY = CACHE_EMBEDDED_MTLS_PREFIX + "-rotation-interval-days";
 
     private static final String CACHE_REMOTE_PREFIX = "cache-remote";
     public static final String CACHE_REMOTE_HOST_PROPERTY = CACHE_REMOTE_PREFIX + "-host";
@@ -102,6 +103,12 @@ public class CachingOptions {
             .description("The password to access the Truststore.")
             .build();
 
+    public static final Option<Integer> CACHE_EMBEDDED_MTLS_ROTATION = new OptionBuilder<>(CACHE_EMBEDDED_MTLS_ROTATION_PROPERTY, Integer.class)
+            .category(OptionCategory.CACHE)
+            .defaultValue(30)
+            .description("Rotation period in days of automatic JGroups MTLS certificates.")
+            .build();
+
     public static final Option<String> CACHE_REMOTE_HOST = new OptionBuilder<>(CACHE_REMOTE_HOST_PROPERTY, String.class)
             .category(OptionCategory.CACHE)
             .description("The hostname of the external Infinispan cluster.")

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/Configuration.java

@@ -145,6 +145,14 @@ public final class Configuration {
         return getOptionalValue(name).map(Boolean::parseBoolean);
     }
 
+    public static Optional<Integer> getOptionalIntegerValue(Option<Integer> option) {
+        return getOptionalIntegerValue(option.getKey());
+    }
+
+    public static Optional<Integer> getOptionalIntegerValue(String propertyName) {
+        return getConfig().getOptionalValue(NS_KEYCLOAK_PREFIX.concat(propertyName), Integer.class);
+    }
+
     public static String getMappedPropertyName(String key) {
         PropertyMapper<?> mapper = PropertyMappers.getMapper(key);
 

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/CachingPropertyMappers.java

@@ -15,6 +15,7 @@ import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.quarkus.runtime.Environment;
 import org.keycloak.quarkus.runtime.cli.PropertyException;
 import org.keycloak.quarkus.runtime.configuration.Configuration;
+import org.keycloak.utils.StringUtil;
 
 import static org.keycloak.quarkus.runtime.configuration.Configuration.getOptionalKcValue;
 import static org.keycloak.quarkus.runtime.configuration.mappers.PropertyMapper.fromOption;
@@ -32,78 +33,83 @@ final class CachingPropertyMappers {
 
     public static PropertyMapper<?>[] getClusteringPropertyMappers() {
         List<PropertyMapper<?>> staticMappers = List.of(
-              fromOption(CachingOptions.CACHE)
-                    .paramLabel("type")
-                    .build(),
-              fromOption(CachingOptions.CACHE_STACK)
-                    .isEnabled(CachingPropertyMappers::cacheSetToInfinispan, CACHE_STACK_SET_TO_ISPN)
-                    .to("kc.spi-connections-infinispan-quarkus-stack")
-                    .paramLabel("stack")
-                    .build(),
-              fromOption(CachingOptions.CACHE_CONFIG_FILE)
-                    .mapFrom(CachingOptions.CACHE, (value, context) -> {
-                        if (CachingOptions.Mechanism.local.name().equals(value)) {
-                            return "cache-local.xml";
-                        } else if (CachingOptions.Mechanism.ispn.name().equals(value)) {
-                            return resolveConfigFile("cache-ispn.xml", null);
-                        } else
-                            return null;
-                    })
-                    .to("kc.spi-connections-infinispan-quarkus-config-file")
-                    .transformer(CachingPropertyMappers::resolveConfigFile)
-                    .paramLabel("file")
-                    .build(),
-              fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED)
-                    .build(),
-              fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE.withRuntimeSpecificDefault(getDefaultKeystorePathValue()))
-                    .paramLabel("file")
-                    .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled.".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
-                    .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE, CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD))
-                    .build(),
-              fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD)
-                    .paramLabel("password")
-                    .isMasked(true)
-                    .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled.".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
-                    .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD, CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE))
-                    .build(),
-              fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE.withRuntimeSpecificDefault(getDefaultTruststorePathValue()))
-                    .paramLabel("file")
-                    .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled.".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
-                    .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE, CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD))
-                    .build(),
-              fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD)
-                    .paramLabel("password")
-                    .isMasked(true)
-                    .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled.".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
-                    .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD, CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE))
-                    .build(),
-              fromOption(CachingOptions.CACHE_REMOTE_HOST)
-                    .paramLabel("hostname")
-                    .addValidateEnabled(CachingPropertyMappers::isRemoteCacheHostEnabled, MULTI_SITE_OR_EMBEDDED_REMOTE_FEATURE_SET)
-                    .isRequired(InfinispanUtils::isRemoteInfinispan, MULTI_SITE_FEATURE_SET)
-                    .build(),
-              fromOption(CachingOptions.CACHE_REMOTE_PORT)
-                    .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
-                    .paramLabel("port")
-                    .build(),
-              fromOption(CachingOptions.CACHE_REMOTE_TLS_ENABLED)
-                    .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
-                    .build(),
-              fromOption(CachingOptions.CACHE_REMOTE_USERNAME)
-                    .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
-                    .validator((value) -> validateCachingOptionIsPresent(CachingOptions.CACHE_REMOTE_USERNAME, CachingOptions.CACHE_REMOTE_PASSWORD))
-                    .paramLabel("username")
-                    .build(),
-              fromOption(CachingOptions.CACHE_REMOTE_PASSWORD)
-                    .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
-                    .validator((value) -> validateCachingOptionIsPresent(CachingOptions.CACHE_REMOTE_PASSWORD, CachingOptions.CACHE_REMOTE_USERNAME))
-                    .paramLabel("password")
-                    .isMasked(true)
-                    .build(),
-              fromOption(CachingOptions.CACHE_METRICS_HISTOGRAMS_ENABLED)
-                    .isEnabled(MetricsPropertyMappers::metricsEnabled, MetricsPropertyMappers.METRICS_ENABLED_MSG)
-                    .build()
-              );
+                fromOption(CachingOptions.CACHE)
+                        .paramLabel("type")
+                        .build(),
+                fromOption(CachingOptions.CACHE_STACK)
+                        .isEnabled(CachingPropertyMappers::cacheSetToInfinispan, CACHE_STACK_SET_TO_ISPN)
+                        .to("kc.spi-connections-infinispan-quarkus-stack")
+                        .paramLabel("stack")
+                        .build(),
+                fromOption(CachingOptions.CACHE_CONFIG_FILE)
+                        .mapFrom(CachingOptions.CACHE, (value, context) -> {
+                            if (CachingOptions.Mechanism.local.name().equals(value)) {
+                                return "cache-local.xml";
+                            } else if (CachingOptions.Mechanism.ispn.name().equals(value)) {
+                                return resolveConfigFile("cache-ispn.xml", null);
+                            } else
+                                return null;
+                        })
+                        .to("kc.spi-connections-infinispan-quarkus-config-file")
+                        .transformer(CachingPropertyMappers::resolveConfigFile)
+                        .paramLabel("file")
+                        .build(),
+                fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED)
+                        .build(),
+                fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE.withRuntimeSpecificDefault(getDefaultKeystorePathValue()))
+                        .paramLabel("file")
+                        .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
+                        .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE, CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD))
+                        .build(),
+                fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD)
+                        .paramLabel("password")
+                        .isMasked(true)
+                        .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
+                        .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD, CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE))
+                        .build(),
+                fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE.withRuntimeSpecificDefault(getDefaultTruststorePathValue()))
+                        .paramLabel("file")
+                        .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
+                        .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE, CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD))
+                        .build(),
+                fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD)
+                        .paramLabel("password")
+                        .isMasked(true)
+                        .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
+                        .validator(value -> checkOptionPresent(CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD, CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE))
+                        .build(),
+                fromOption(CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION)
+                        .paramLabel("days")
+                        .isEnabled(() -> Configuration.isTrue(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED), "property '%s' is enabled".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ENABLED.getKey()))
+                        .validator(CachingPropertyMappers::validateCertificateRotationIsPositive)
+                        .build(),
+                fromOption(CachingOptions.CACHE_REMOTE_HOST)
+                        .paramLabel("hostname")
+                        .addValidateEnabled(CachingPropertyMappers::isRemoteCacheHostEnabled, MULTI_SITE_OR_EMBEDDED_REMOTE_FEATURE_SET)
+                        .isRequired(InfinispanUtils::isRemoteInfinispan, MULTI_SITE_FEATURE_SET)
+                        .build(),
+                fromOption(CachingOptions.CACHE_REMOTE_PORT)
+                        .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
+                        .paramLabel("port")
+                        .build(),
+                fromOption(CachingOptions.CACHE_REMOTE_TLS_ENABLED)
+                        .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
+                        .build(),
+                fromOption(CachingOptions.CACHE_REMOTE_USERNAME)
+                        .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
+                        .validator((value) -> validateCachingOptionIsPresent(CachingOptions.CACHE_REMOTE_USERNAME, CachingOptions.CACHE_REMOTE_PASSWORD))
+                        .paramLabel("username")
+                        .build(),
+                fromOption(CachingOptions.CACHE_REMOTE_PASSWORD)
+                        .isEnabled(CachingPropertyMappers::remoteHostSet, CachingPropertyMappers.REMOTE_HOST_SET)
+                        .validator((value) -> validateCachingOptionIsPresent(CachingOptions.CACHE_REMOTE_PASSWORD, CachingOptions.CACHE_REMOTE_USERNAME))
+                        .paramLabel("password")
+                        .isMasked(true)
+                        .build(),
+                fromOption(CachingOptions.CACHE_METRICS_HISTOGRAMS_ENABLED)
+                        .isEnabled(MetricsPropertyMappers::metricsEnabled, MetricsPropertyMappers.METRICS_ENABLED_MSG)
+                        .build()
+        );
 
         int numMappers = staticMappers.size() + CachingOptions.LOCAL_MAX_COUNT_CACHES.length + CachingOptions.CLUSTERED_MAX_COUNT_CACHES.length;
         List<PropertyMapper<?>> mappers = new ArrayList<>(numMappers);
@@ -165,9 +171,9 @@ final class CachingPropertyMappers {
 
     private static PropertyMapper<?> maxCountOpt(String cacheName, BooleanSupplier isEnabled, String enabledWhen) {
         return fromOption(CachingOptions.maxCountOption(cacheName))
-              .isEnabled(isEnabled, enabledWhen)
-              .paramLabel("max-count")
-              .build();
+                .isEnabled(isEnabled, enabledWhen)
+                .paramLabel("max-count")
+                .build();
     }
 
     private static boolean isRemoteCacheHostEnabled() {
@@ -186,4 +192,18 @@ final class CachingPropertyMappers {
         }
         throw new PropertyException("The option '%s' requires '%s' to be enabled.".formatted(option.getKey(), requiredOption.getKey()));
     }
+
+    private static void validateCertificateRotationIsPositive(String value) {
+        value = value.trim();
+        if (StringUtil.isBlank(value)) {
+            throw new PropertyException("Option '%s' must not be empty.".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION.getKey()));
+        }
+        try {
+            if (Integer.parseInt(value) <= 0) {
+                throw new NumberFormatException();
+            }
+        } catch (NumberFormatException unused) {
+            throw new PropertyException("JGroups MTLS certificate rotation in '%s' option must positive.".formatted(CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION.getKey()));
+        }
+    }
 }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java

@@ -58,6 +58,7 @@ import org.keycloak.common.Profile;
 import org.keycloak.common.util.MultiSiteUtils;
 import org.keycloak.config.CachingOptions;
 import org.keycloak.config.MetricsOptions;
+import org.keycloak.config.Option;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.InfinispanUtil;
 import org.keycloak.infinispan.util.InfinispanUtils;
@@ -531,6 +532,11 @@ public class CacheManagerFactory {
         return Configuration.getOptionalKcValue(propertyName).orElseThrow(() -> new RuntimeException("Property " + propertyName + " required but not specified"));
     }
 
+    public static int requiredIntegerProperty(Option<Integer> option) {
+        return Configuration.getOptionalIntegerValue(option)
+                .orElseThrow(() -> new RuntimeException("Property '%s' required but not specified".formatted(option.getKey())));
+    }
+
     private static boolean hasRemoteStore(ConfigurationBuilder builder) {
         return builder.persistence().stores().stream().anyMatch(RemoteStoreConfigurationBuilder.class::isInstance);
     }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/jgroups/impl/BaseJGroupsTlsConfigurator.java

@@ -29,11 +29,11 @@ abstract class BaseJGroupsTlsConfigurator implements JGroupsStackConfigurator {
 
     @Override
     public void configure(ConfigurationBuilderHolder holder, KeycloakSession session) {
-        var factory = createSocketFactory(session);
+        var factory = createSocketFactory(holder, session);
         JGroupsUtil.transportOf(holder).addProperty(JGroupsTransport.SOCKET_FACTORY, factory);
         JGroupsUtil.validateTlsAvailable(holder);
         CacheManagerFactory.logger.info("JGroups Encryption enabled (mTLS).");
     }
 
-    abstract SocketFactory createSocketFactory(KeycloakSession session);
+    abstract SocketFactory createSocketFactory(ConfigurationBuilderHolder holder, KeycloakSession session);
 }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/jgroups/impl/FileJGroupsTlsConfigurator.java

@@ -17,6 +17,8 @@
 
 package org.keycloak.quarkus.runtime.storage.infinispan.jgroups.impl;
 
+import org.infinispan.configuration.parsing.ConfigurationBuilderHolder;
+import org.jgroups.util.FileWatcher;
 import org.jgroups.util.SocketFactory;
 import org.jgroups.util.TLS;
 import org.jgroups.util.TLSClientAuth;
@@ -36,7 +38,7 @@ public class FileJGroupsTlsConfigurator extends BaseJGroupsTlsConfigurator {
     public static final FileJGroupsTlsConfigurator INSTANCE = new FileJGroupsTlsConfigurator();
 
     @Override
-    SocketFactory createSocketFactory(KeycloakSession ignored) {
+    SocketFactory createSocketFactory(ConfigurationBuilderHolder holder, KeycloakSession ignored) {
         var tls = new TLS()
                 .enabled(true)
                 .setKeystorePath(requiredStringProperty(CACHE_EMBEDDED_MTLS_KEYSTORE_FILE_PROPERTY))
@@ -47,6 +49,8 @@ public class FileJGroupsTlsConfigurator extends BaseJGroupsTlsConfigurator {
                 .setTruststoreType("pkcs12")
                 .setClientAuth(TLSClientAuth.NEED)
                 .setProtocols(new String[]{"TLSv1.3"});
+        // listen to file changes and reloads the key and trust stores.
+        tls.setWatcher(new FileWatcher());
         return tls.createSocketFactory();
     }
 

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/jgroups/impl/JpaJGroupsTlsConfigurator.java

@@ -19,41 +19,36 @@ package org.keycloak.quarkus.runtime.storage.infinispan.jgroups.impl;
 
 import java.io.IOException;
 import java.security.GeneralSecurityException;
-import java.security.KeyPair;
-import java.security.cert.X509Certificate;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import org.infinispan.configuration.parsing.ConfigurationBuilderHolder;
 import org.jgroups.util.DefaultSocketFactory;
 import org.jgroups.util.SocketFactory;
-import org.keycloak.common.crypto.CryptoIntegration;
-import org.keycloak.common.util.CertificateUtils;
-import org.keycloak.common.util.KeyUtils;
-import org.keycloak.common.util.KeystoreUtil;
 import org.keycloak.common.util.Retry;
+import org.keycloak.config.CachingOptions;
+import org.keycloak.infinispan.module.certificates.CertificateReloadManager;
+import org.keycloak.infinispan.module.certificates.JGroupsCertificateHolder;
+import org.keycloak.infinispan.module.configuration.global.KeycloakConfigurationBuilder;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.storage.configuration.ServerConfigStorageProvider;
-import org.keycloak.util.JsonSerialization;
 
 import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLServerSocket;
 import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509ExtendedKeyManager;
-import javax.net.ssl.X509ExtendedTrustManager;
+
+import static org.keycloak.infinispan.module.certificates.CertificateReloadManager.CERTIFICATE_ID;
+import static org.keycloak.infinispan.module.certificates.JGroupsCertificate.fromJson;
+import static org.keycloak.quarkus.runtime.storage.infinispan.CacheManagerFactory.requiredIntegerProperty;
 
 /**
  * JGroups mTLS configuration using certificates stored by {@link ServerConfigStorageProvider}.
  */
 public class JpaJGroupsTlsConfigurator extends BaseJGroupsTlsConfigurator {
 
-    private static final char[] KEY_PASSWORD = "jgroups-password".toCharArray();
-    private static final String CERTIFICATE_ID = "crt_jgroups";
-    private static final String KEYSTORE_ALIAS = "jgroups";
-    private static final String JGROUPS_SUBJECT = "jgroups";
     private static final String TLS_PROTOCOL_VERSION = "TLSv1.3";
     private static final String TLS_PROTOCOL = "TLS";
     private static final int STARTUP_RETRIES = 2;
@@ -66,62 +61,36 @@ public class JpaJGroupsTlsConfigurator extends BaseJGroupsTlsConfigurator {
     }
 
     @Override
-    SocketFactory createSocketFactory(KeycloakSession session) {
+    SocketFactory createSocketFactory(ConfigurationBuilderHolder holder, KeycloakSession session) {
         var factory = session.getKeycloakSessionFactory();
-        return Retry.call(iteration -> KeycloakModelUtils.runJobInTransactionWithResult(factory, this::createSocketFactoryInTransaction), STARTUP_RETRIES, STARTUP_RETRY_SLEEP_MILLIS);
+        var kcConfig = holder.getGlobalConfigurationBuilder().addModule(KeycloakConfigurationBuilder.class);
+        kcConfig.setKeycloakSessionFactory(factory);
+        kcConfig.setJGroupsCertificateRotation(requiredIntegerProperty(CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION));
+        return Retry.call(iteration -> {
+            try {
+                var crtHolder = KeycloakModelUtils.runJobInTransactionWithResult(factory, this::createSocketFactoryInTransaction);
+                var sslContext = SSLContext.getInstance(TLS_PROTOCOL);
+                sslContext.init(new KeyManager[]{crtHolder.keyManager()}, new TrustManager[]{crtHolder.trustManager()}, null);
+                var sf = createFromContext(sslContext);
+                kcConfig.setJGroupCertificateHolder(crtHolder);
+                return sf;
+            } catch (KeyManagementException | NoSuchAlgorithmException e) {
+                throw new RuntimeException(e);
+            }
+        }, STARTUP_RETRIES, STARTUP_RETRY_SLEEP_MILLIS);
     }
 
-    private SocketFactory createSocketFactoryInTransaction(KeycloakSession session) {
+    private JGroupsCertificateHolder createSocketFactoryInTransaction(KeycloakSession session) {
         try {
+            var rotationDays = requiredIntegerProperty(CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION);
             var storage = session.getProvider(ServerConfigStorageProvider.class);
-            var data = fromJson(storage.loadOrCreate(CERTIFICATE_ID, JpaJGroupsTlsConfigurator::generateSelfSignedCertificate));
-            var km = createKeyManager(data.getKeyPair(), data.getCertificate());
-            var tm = createTrustManager(data.getCertificate());
-            var sslContext = SSLContext.getInstance(TLS_PROTOCOL);
-            sslContext.init(new KeyManager[]{km}, new TrustManager[]{tm}, null);
-            return createFromContext(sslContext);
+            var data = fromJson(storage.loadOrCreate(CERTIFICATE_ID, () -> CertificateReloadManager.generateSelfSignedCertificate(rotationDays * 2L)));
+            return JGroupsCertificateHolder.create(data);
         } catch (IOException | GeneralSecurityException e) {
             throw new RuntimeException(e);
         }
     }
 
-    private X509ExtendedKeyManager createKeyManager(KeyPair keyPair, X509Certificate certificate) throws GeneralSecurityException, IOException {
-        var ks = CryptoIntegration.getProvider().getKeyStore(KeystoreUtil.KeystoreFormat.JKS);
-        ks.load(null, null);
-        ks.setKeyEntry(KEYSTORE_ALIAS, keyPair.getPrivate(), KEY_PASSWORD, new java.security.cert.Certificate[]{certificate});
-        var kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-        kmf.init(ks, KEY_PASSWORD);
-        for (KeyManager km : kmf.getKeyManagers()) {
-            if (km instanceof X509ExtendedKeyManager) {
-                return (X509ExtendedKeyManager) km;
-            }
-        }
-        throw new GeneralSecurityException("Could not obtain an X509ExtendedKeyManager");
-    }
-
-    private X509ExtendedTrustManager createTrustManager(X509Certificate certificate) throws GeneralSecurityException, IOException {
-        var ks = CryptoIntegration.getProvider().getKeyStore(KeystoreUtil.KeystoreFormat.JKS);
-        ks.load(null, null);
-        ks.setCertificateEntry(KEYSTORE_ALIAS, certificate);
-        var tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-        tmf.init(ks);
-        for (TrustManager tm : tmf.getTrustManagers()) {
-            if (tm instanceof X509ExtendedTrustManager) {
-                return (X509ExtendedTrustManager) tm;
-            }
-        }
-        throw new GeneralSecurityException("Could not obtain an X509TrustManager");
-    }
-
-    private static String generateSelfSignedCertificate() {
-        var keyPair = KeyUtils.generateRsaKeyPair(2048);
-        var certificate = CertificateUtils.generateV1SelfSignedCertificate(keyPair, JGROUPS_SUBJECT);
-        var entity = new CertificateEntity();
-        entity.setCertificate(certificate);
-        entity.setKeyPair(keyPair);
-        return toJson(entity);
-    }
-
     private static SocketFactory createFromContext(SSLContext context) {
         DefaultSocketFactory socketFactory = new DefaultSocketFactory(context);
         final SSLParameters serverParameters = new SSLParameters();
@@ -131,20 +100,4 @@ public class JpaJGroupsTlsConfigurator extends BaseJGroupsTlsConfigurator {
         return socketFactory;
     }
 
-    private static String toJson(CertificateEntity entity) {
-        try {
-            return JsonSerialization.mapper.writeValueAsString(entity);
-        } catch (JsonProcessingException e) {
-            throw new IllegalStateException("Should never happen!", e);
-        }
-    }
-
-    private static CertificateEntity fromJson(String json) {
-        try {
-            return JsonSerialization.mapper.readValue(json, CertificateEntity.class);
-        } catch (JsonProcessingException e) {
-            throw new IllegalStateException("Should never happen!", e);
-        }
-    }
-
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/CacheEmbeddedMtlsDistTest.java

@@ -38,10 +38,11 @@ public class CacheEmbeddedMtlsDistTest {
                 CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE,
                 CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE,
                 CachingOptions.CACHE_EMBEDDED_MTLS_KEYSTORE_PASSWORD,
-                CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD
+                CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD,
+                CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION
         )) {
-            var result = dist.run("start-dev", "--cache=ispn", "--%s=a".formatted(option.getKey()));
-            result.assertError("Disabled option: '--%s'. Available only when property 'cache-embedded-mtls-enabled' is enabled.".formatted(option.getKey()));
+            var result = dist.run("start-dev", "--cache=ispn", "--%s=1".formatted(option.getKey()));
+            result.assertError("Disabled option: '--%s'. Available only when property 'cache-embedded-mtls-enabled' is enabled".formatted(option.getKey()));
         }
     }
 
@@ -53,6 +54,24 @@ public class CacheEmbeddedMtlsDistTest {
         doFileAndPasswordValidation(dist, CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE, CachingOptions.CACHE_EMBEDDED_MTLS_TRUSTSTORE_PASSWORD);
     }
 
+    @DryRun
+    @Test
+    @RawDistOnly(reason = "Containers are immutable")
+    public void testCacheEmbeddedMtlsValidation(KeycloakDistribution dist) {
+        var key = CachingOptions.CACHE_EMBEDDED_MTLS_ROTATION.getKey();
+        // test zero
+        var result = dist.run("start-dev", "--cache=ispn", "--cache-embedded-mtls-enabled=true", "--%s=0".formatted(key));
+        result.assertError("JGroups MTLS certificate rotation in '%s' option must positive.".formatted(key));
+
+        // test negative
+        result = dist.run("start-dev", "--cache=ispn", "--cache-embedded-mtls-enabled=true", "--%s=-1".formatted(key));
+        result.assertError("JGroups MTLS certificate rotation in '%s' option must positive.".formatted(key));
+
+        // test blank
+        result = dist.run("start-dev", "--cache=ispn", "--cache-embedded-mtls-enabled=true", "--%s=".formatted(key));
+        result.assertError("Invalid value for option '--%s': '' is not an int".formatted(key));
+    }
+
     @Test
     @RawDistOnly(reason = "Containers are immutable")
     public void testCacheEmbeddedMtlsEnabled(KeycloakDistribution dist) {

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HelpCommandDistTest.java

@@ -204,7 +204,7 @@ public class HelpCommandDistTest {
 
         try {
             Approvals.verify(output);
-        } catch (AssertionError cause) {
+        } catch (Error cause) {
             if ("true".equals(System.getenv(REPLACE_EXPECTED))) {
                 try {
                     FileUtils.write(Approvals.createApprovalNamer().getApprovedFile(".txt"), output,

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelpAll.approved.txt

@@ -42,18 +42,21 @@ Cache:
                      The Keystore file path. The Keystore must contain the certificate to use by
                        the TLS protocol. By default, it lookup 'cache-mtls-keystore.p12' under
                        conf/ directory. Available only when property 'cache-embedded-mtls-enabled'
-                       is enabled..
+                       is enabled.
 --cache-embedded-mtls-key-store-password <password>
                      The password to access the Keystore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
+--cache-embedded-mtls-rotation-interval-days <days>
+                     Rotation period in days of automatic JGroups MTLS certificates. Default: 30.
+                       Available only when property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-file <file>
                      The Truststore file path. It should contain the trusted certificates or the
                        Certificate Authority that signed the certificates. By default, it lookup
                        'cache-mtls-truststore.p12' under conf/ directory. Available only when
-                       property 'cache-embedded-mtls-enabled' is enabled..
+                       property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-password <password>
                      The password to access the Truststore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-offline-client-sessions-max-count <max-count>
                      The maximum number of entries that can be stored in-memory by the
                        offlineClientSessions cache. Available only when embedded Infinispan

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelpAll.approved.txt

@@ -43,18 +43,21 @@ Cache:
                      The Keystore file path. The Keystore must contain the certificate to use by
                        the TLS protocol. By default, it lookup 'cache-mtls-keystore.p12' under
                        conf/ directory. Available only when property 'cache-embedded-mtls-enabled'
-                       is enabled..
+                       is enabled.
 --cache-embedded-mtls-key-store-password <password>
                      The password to access the Keystore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
+--cache-embedded-mtls-rotation-interval-days <days>
+                     Rotation period in days of automatic JGroups MTLS certificates. Default: 30.
+                       Available only when property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-file <file>
                      The Truststore file path. It should contain the trusted certificates or the
                        Certificate Authority that signed the certificates. By default, it lookup
                        'cache-mtls-truststore.p12' under conf/ directory. Available only when
-                       property 'cache-embedded-mtls-enabled' is enabled..
+                       property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-password <password>
                      The password to access the Truststore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-offline-client-sessions-max-count <max-count>
                      The maximum number of entries that can be stored in-memory by the
                        offlineClientSessions cache. Available only when embedded Infinispan

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelpAll.approved.txt

@@ -43,18 +43,21 @@ Cache:
                      The Keystore file path. The Keystore must contain the certificate to use by
                        the TLS protocol. By default, it lookup 'cache-mtls-keystore.p12' under
                        conf/ directory. Available only when property 'cache-embedded-mtls-enabled'
-                       is enabled..
+                       is enabled.
 --cache-embedded-mtls-key-store-password <password>
                      The password to access the Keystore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
+--cache-embedded-mtls-rotation-interval-days <days>
+                     Rotation period in days of automatic JGroups MTLS certificates. Default: 30.
+                       Available only when property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-file <file>
                      The Truststore file path. It should contain the trusted certificates or the
                        Certificate Authority that signed the certificates. By default, it lookup
                        'cache-mtls-truststore.p12' under conf/ directory. Available only when
-                       property 'cache-embedded-mtls-enabled' is enabled..
+                       property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-password <password>
                      The password to access the Truststore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-offline-client-sessions-max-count <max-count>
                      The maximum number of entries that can be stored in-memory by the
                        offlineClientSessions cache. Available only when embedded Infinispan

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testUpdateCompatibilityCheckHelpAll.approved.txt

@@ -42,18 +42,21 @@ Cache:
                      The Keystore file path. The Keystore must contain the certificate to use by
                        the TLS protocol. By default, it lookup 'cache-mtls-keystore.p12' under
                        conf/ directory. Available only when property 'cache-embedded-mtls-enabled'
-                       is enabled..
+                       is enabled.
 --cache-embedded-mtls-key-store-password <password>
                      The password to access the Keystore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
+--cache-embedded-mtls-rotation-interval-days <days>
+                     Rotation period in days of automatic JGroups MTLS certificates. Default: 30.
+                       Available only when property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-file <file>
                      The Truststore file path. It should contain the trusted certificates or the
                        Certificate Authority that signed the certificates. By default, it lookup
                        'cache-mtls-truststore.p12' under conf/ directory. Available only when
-                       property 'cache-embedded-mtls-enabled' is enabled..
+                       property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-password <password>
                      The password to access the Truststore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-offline-client-sessions-max-count <max-count>
                      The maximum number of entries that can be stored in-memory by the
                        offlineClientSessions cache. Available only when embedded Infinispan

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testUpdateCompatibilityMetadataHelpAll.approved.txt

@@ -40,18 +40,21 @@ Cache:
                      The Keystore file path. The Keystore must contain the certificate to use by
                        the TLS protocol. By default, it lookup 'cache-mtls-keystore.p12' under
                        conf/ directory. Available only when property 'cache-embedded-mtls-enabled'
-                       is enabled..
+                       is enabled.
 --cache-embedded-mtls-key-store-password <password>
                      The password to access the Keystore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
+--cache-embedded-mtls-rotation-interval-days <days>
+                     Rotation period in days of automatic JGroups MTLS certificates. Default: 30.
+                       Available only when property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-file <file>
                      The Truststore file path. It should contain the trusted certificates or the
                        Certificate Authority that signed the certificates. By default, it lookup
                        'cache-mtls-truststore.p12' under conf/ directory. Available only when
-                       property 'cache-embedded-mtls-enabled' is enabled..
+                       property 'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-mtls-trust-store-password <password>
                      The password to access the Truststore. Available only when property
-                       'cache-embedded-mtls-enabled' is enabled..
+                       'cache-embedded-mtls-enabled' is enabled.
 --cache-embedded-offline-client-sessions-max-count <max-count>
                      The maximum number of entries that can be stored in-memory by the
                        offlineClientSessions cache. Available only when embedded Infinispan

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/JGroupsCertificateRotationClusterTest.java

@@ -0,0 +1,172 @@
+package org.keycloak.testsuite.cluster;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import org.awaitility.Awaitility;
+import org.infinispan.factories.GlobalComponentRegistry;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.junit.Assume;
+import org.junit.Test;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.infinispan.module.certificates.CertificateReloadManager;
+import org.keycloak.models.KeycloakSession;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+public class JGroupsCertificateRotationClusterTest extends AbstractClusterTest {
+
+    @Test
+    public void testRotation() {
+        Assume.assumeTrue(getClusterSize() >= 2);
+        var mtlsEnabled = assumeEnabledAndOverwriteRotation(1, TimeUnit.DAYS);
+        Assume.assumeTrue(mtlsEnabled);
+        assertClusterSize();
+
+        var alias = currentCertificateAliasFor(0);
+        log.infof("Current JGroups Certificate alias: %s", alias);
+
+        // test rotation in all the nodes
+        for (int i = 0; i < getClusterSize(); ++i) {
+            rotateCertificate(i);
+            assertAliasNotEquals(alias);
+
+            alias = currentCertificateAliasFor(i);
+            log.infof("Current JGroups Certificate alias after rotation: %s", alias);
+        }
+    }
+
+    @Test
+    public void testAutoRotation() {
+        Assume.assumeTrue(getClusterSize() >= 2);
+        var mtlsEnabled = assumeEnabledAndOverwriteRotation(5, TimeUnit.SECONDS);
+        Assume.assumeTrue(mtlsEnabled);
+        assertClusterSize();
+
+        var alias = currentCertificateAliasFor(0);
+        log.infof("Current JGroups Certificate alias: %s", alias);
+
+        // The certificate should rotate after 5 seconds
+        assertAliasNotEquals(alias);
+    }
+
+    @Test
+    public void testCoordinatorHasScheduleTask() {
+        Assume.assumeTrue(getClusterSize() >= 2);
+        var mtlsEnabled = assumeEnabledAndOverwriteRotation(1, TimeUnit.DAYS);
+        Assume.assumeTrue(mtlsEnabled);
+
+        var alias = currentCertificateAliasFor(0);
+        log.infof("Current JGroups Certificate alias: %s", alias);
+
+        int coordinatorIdx = -1;
+        for (int i = 0; i < getClusterSize(); ++i) {
+            if (isCoordinator(i)) {
+                assertTrue(hasRotationTask(i));
+                coordinatorIdx = i;
+                break;
+            }
+        }
+
+        assertTrue(coordinatorIdx >= 0);
+        killBackendNode(backendNode(coordinatorIdx));
+        failback();
+        assertClusterSize();
+
+        // new coordinator should be the next in line
+        coordinatorIdx++;
+        if (coordinatorIdx >= getClusterSize()) {
+            coordinatorIdx = 0;
+        }
+
+        assertTrue(isCoordinator(coordinatorIdx));
+        assertTrue(hasRotationTask(coordinatorIdx));
+    }
+
+    private boolean assumeEnabledAndOverwriteRotation(long time, TimeUnit timeUnit) {
+        boolean enabled = false;
+        for (int i = 0; i < getClusterSize(); ++i) {
+            var crmEnabled = getTestingClientFor(backendNode(i))
+                    .server()
+                    .fetch(session -> {
+                        var crm = certificateReloadManager(session);
+                        if (crm == null) {
+                            return false;
+                        }
+                        crm.setRotationSeconds(timeUnit.toSeconds(time));
+                        if (crm.isCoordinator()) {
+                            crm.rotateCertificate();
+                        }
+                        return true;
+                    }, Boolean.class);
+            if (crmEnabled) {
+                enabled = true;
+            }
+        }
+        return enabled;
+    }
+
+    private void assertAliasNotEquals(String alias) {
+        for (int i = 0; i < getClusterSize(); ++i) {
+            int nodeIdx = i;
+            Awaitility.waitAtMost(Duration.ofMinutes(1))
+                    .pollDelay(Duration.ofSeconds(1))
+                    .untilAsserted(() -> assertNotEquals(alias, currentCertificateAliasFor(nodeIdx)));
+        }
+    }
+
+    private String currentCertificateAliasFor(int index) {
+        return getTestingClientFor(backendNode(index)).server().fetch(JGroupsCertificateRotationClusterTest::currentCertificateAlias, String.class);
+    }
+
+    private void rotateCertificate(int index) {
+        getTestingClientFor(backendNode(index)).server().run(JGroupsCertificateRotationClusterTest::rotateCertificate);
+    }
+
+    private boolean isCoordinator(int index) {
+        return getTestingClientFor(backendNode(index)).server().fetch(session -> certificateReloadManager(session).isCoordinator(), Boolean.class);
+    }
+
+    private boolean hasRotationTask(int index) {
+        return getTestingClientFor(backendNode(index)).server().fetch(session -> certificateReloadManager(session).hasRotationTask(), Boolean.class);
+    }
+
+    private int fetchClusterSize(int index) {
+        return getTestingClientFor(backendNode(index)).server().fetch(session -> cacheManager(session).getMembers().size(), Integer.class);
+    }
+
+    private void assertClusterSize(){
+        var expectedSize = getClusterSize();
+        for (int i = 0; i < expectedSize; ++i) {
+            var nodeIndex = i;
+            Awaitility.waitAtMost(Duration.ofMinutes(1))
+                    .pollDelay(Duration.ofSeconds(1))
+                    .untilAsserted(() -> assertEquals(expectedSize, fetchClusterSize(nodeIndex)));
+        }
+    }
+
+    private static CertificateReloadManager certificateReloadManager(KeycloakSession session) {
+        return GlobalComponentRegistry.componentOf(cacheManager(session), CertificateReloadManager.class);
+    }
+
+    private static EmbeddedCacheManager cacheManager(KeycloakSession session) {
+        return session.getProvider(InfinispanConnectionProvider.class)
+                .getCache(InfinispanConnectionProvider.USER_CACHE_NAME)
+                .getCacheManager();
+    }
+
+    private static String currentCertificateAlias(KeycloakSession session) {
+        return certificateReloadManager(session)
+                .currentCertificate()
+                .getAlias();
+    }
+
+    private static void rotateCertificate(KeycloakSession session) {
+        certificateReloadManager(session).rotateCertificate();
+    }
+
+}
+
+