From 856df9ea3db5952ed209be53b87da5be0eea84cf Mon Sep 17 00:00:00 2001 From: Steven Hawkins Date: Thu, 28 Aug 2025 05:05:16 -0400 Subject: [PATCH] fix: adds simple log scraping to error state detection (#41800) closes: #21816 Signed-off-by: Steve Hawkins --- .../controllers/KeycloakController.java | 17 +++++++++++++++ operator/src/main/kubernetes/kubernetes.yml | 6 ++++++ .../integration/KeycloakDeploymentTest.java | 21 +++++++++++++++++++ .../operator/testsuite/utils/CRAssert.java | 4 ++-- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/operator/src/main/java/org/keycloak/operator/controllers/KeycloakController.java b/operator/src/main/java/org/keycloak/operator/controllers/KeycloakController.java index 83c04d861b8..30546a66875 100644 --- a/operator/src/main/java/org/keycloak/operator/controllers/KeycloakController.java +++ b/operator/src/main/java/org/keycloak/operator/controllers/KeycloakController.java @@ -22,6 +22,7 @@ import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.PodSpec; import io.fabric8.kubernetes.api.model.PodStatus; import io.fabric8.kubernetes.api.model.apps.StatefulSet; +import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.readiness.Readiness; import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; import io.fabric8.kubernetes.client.utils.Serialization; @@ -284,6 +285,14 @@ public class KeycloakController implements Reconciler { if (Optional.ofNullable(cs.getState()).map(ContainerState::getWaiting) .map(ContainerStateWaiting::getReason).map(String::toLowerCase) .filter(s -> s.contains("err") || s.equals("crashloopbackoff")).isPresent()) { + // since we've failed, try to get the previous first, then the current + String log = null; + try { + log = context.getClient().raw(String.format("/api/v1/namespaces/%s/pods/%s/log?previous=true&tailLines=200", p.getMetadata().getNamespace(), p.getMetadata().getName())); + } catch (KubernetesClientException e) { + // just ignore + } + Log.infof("Found unhealthy container on pod %s/%s: %s", p.getMetadata().getNamespace(), p.getMetadata().getName(), Serialization.asYaml(cs)); @@ -291,6 +300,14 @@ public class KeycloakController implements Reconciler { String.format("Waiting for %s/%s due to %s: %s", p.getMetadata().getNamespace(), p.getMetadata().getName(), cs.getState().getWaiting().getReason(), cs.getState().getWaiting().getMessage())); + if (log != null) { + if (log.length() > 2000) { + log = "... " + log.substring(log.length() - 2000, log.length()); + } + status.addErrorMessage( + String.format("Log for %s/%s: %s", p.getMetadata().getNamespace(), + p.getMetadata().getName(), log)); + } } }); }); diff --git a/operator/src/main/kubernetes/kubernetes.yml b/operator/src/main/kubernetes/kubernetes.yml index 888c2068dfb..685e2c2f287 100644 --- a/operator/src/main/kubernetes/kubernetes.yml +++ b/operator/src/main/kubernetes/kubernetes.yml @@ -42,6 +42,12 @@ rules: - pods verbs: - list + - apiGroups: + - "" + resources: + - pods/log + verbs: + - get - apiGroups: - batch resources: diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java index d65837b96cf..0de9251cff8 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java @@ -32,6 +32,7 @@ import io.fabric8.kubernetes.client.dsl.Resource; import io.quarkus.logging.Log; import io.quarkus.test.junit.QuarkusTest; +import org.assertj.core.api.Condition; import org.awaitility.Awaitility; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Tag; @@ -46,6 +47,7 @@ import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak; import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusCondition; import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret; import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec; +import org.keycloak.operator.crds.v2alpha1.deployment.spec.FeatureSpecBuilder; import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpecBuilder; import org.keycloak.operator.testsuite.apiserver.DisabledIfApiServerTest; import org.keycloak.operator.testsuite.unit.WatchedResourcesTest; @@ -545,6 +547,25 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { }); } + @Test + public void testConfigErrorLog() { + var kc = getTestKeycloakDeployment(true); + kc.getSpec().setFeatureSpec(new FeatureSpecBuilder().addToEnabledFeatures("feature doesn't exist").build()); + + deployKeycloak(k8sclient, kc, false); + + var crSelector = k8sclient.resource(kc); + + Awaitility.await().atMost(3, MINUTES).pollDelay(1, SECONDS).ignoreExceptions().untilAsserted(() -> { + Keycloak current = crSelector.get(); + CRAssert.assertKeycloakStatusCondition(current, KeycloakStatusCondition.READY, false); + CRAssert.assertKeycloakStatusCondition(current, KeycloakStatusCondition.HAS_ERRORS, true, null).has(new Condition<>( + c -> c.getMessage().contains(String.format("Waiting for %s/%s-0 due to CrashLoopBackOff", k8sclient.getNamespace(), kc.getMetadata().getName())) + && c.getMessage().contains("feature doesn't exist"), "message" + )); + }); + } + @Test public void testHttpRelativePathWithPlainValue() { var kc = getTestKeycloakDeployment(false); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/utils/CRAssert.java b/operator/src/test/java/org/keycloak/operator/testsuite/utils/CRAssert.java index dfff37b2e66..9f08017c584 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/utils/CRAssert.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/utils/CRAssert.java @@ -75,10 +75,10 @@ public final class CRAssert { public static void assertKeycloakStatusCondition(Keycloak kc, String condition, Boolean status) { assertKeycloakStatusCondition(kc, condition, status, null); } - public static void assertKeycloakStatusCondition(Keycloak kc, String condition, Boolean status, String containedMessage) { + public static ObjectAssert assertKeycloakStatusCondition(Keycloak kc, String condition, Boolean status, String containedMessage) { Log.debugf("Asserting CR: %s, condition: %s, status: %s, message: %s", kc.getMetadata().getName(), condition, status, containedMessage); try { - assertKeycloakStatusCondition(kc.getStatus(), condition, status, containedMessage, null); + return assertKeycloakStatusCondition(kc.getStatus(), condition, status, containedMessage, null); } catch (Exception e) { Log.infof("Asserting CR: %s with status:\n%s", kc.getMetadata().getName(), Serialization.asYaml(kc.getStatus())); throw e;