From 20e78e468dccb8b89fc920d27aed2ecfbb185b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Vacek?= <86605314+vaceksimon@users.noreply.github.com> Date: Mon, 9 Feb 2026 06:51:31 +0100 Subject: [PATCH] Test framework validations and error messages (#45869) Closes #38163 Signed-off-by: Simon Vacek --- .../FatalTestClassException.java | 9 +++ .../testframework/TestFrameworkException.java | 9 --- .../admin/AdminClientSupplier.java | 8 +-- .../testframework/injection/Registry.java | 63 +++++++++++++++---- .../testframework/injection/RegistryTest.java | 26 +++++++- 5 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 test-framework/core/src/main/java/org/keycloak/testframework/FatalTestClassException.java delete mode 100644 test-framework/core/src/main/java/org/keycloak/testframework/TestFrameworkException.java diff --git a/test-framework/core/src/main/java/org/keycloak/testframework/FatalTestClassException.java b/test-framework/core/src/main/java/org/keycloak/testframework/FatalTestClassException.java new file mode 100644 index 00000000000..5da917fa9dc --- /dev/null +++ b/test-framework/core/src/main/java/org/keycloak/testframework/FatalTestClassException.java @@ -0,0 +1,9 @@ +package org.keycloak.testframework; + +public class FatalTestClassException extends RuntimeException { + + public FatalTestClassException(String message) { + super(message); + } + +} diff --git a/test-framework/core/src/main/java/org/keycloak/testframework/TestFrameworkException.java b/test-framework/core/src/main/java/org/keycloak/testframework/TestFrameworkException.java deleted file mode 100644 index 6e718e96e34..00000000000 --- a/test-framework/core/src/main/java/org/keycloak/testframework/TestFrameworkException.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.keycloak.testframework; - -public class TestFrameworkException extends RuntimeException { - - public TestFrameworkException(String message) { - super(message); - } - -} diff --git a/test-framework/core/src/main/java/org/keycloak/testframework/admin/AdminClientSupplier.java b/test-framework/core/src/main/java/org/keycloak/testframework/admin/AdminClientSupplier.java index a58b97e6b6f..4922543a614 100644 --- a/test-framework/core/src/main/java/org/keycloak/testframework/admin/AdminClientSupplier.java +++ b/test-framework/core/src/main/java/org/keycloak/testframework/admin/AdminClientSupplier.java @@ -7,7 +7,7 @@ import org.keycloak.admin.client.Keycloak; import org.keycloak.representations.idm.ClientRepresentation; import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.representations.idm.UserRepresentation; -import org.keycloak.testframework.TestFrameworkException; +import org.keycloak.testframework.FatalTestClassException; import org.keycloak.testframework.annotations.InjectAdminClient; import org.keycloak.testframework.config.Config; import org.keycloak.testframework.injection.DependenciesBuilder; @@ -49,20 +49,20 @@ public class AdminClientSupplier implements Supplier c.getClientId().equals(annotation.client())) - .findFirst().orElseThrow(() -> new TestFrameworkException("Client " + annotation.client() + " not found in managed realm")); + .findFirst().orElseThrow(() -> new FatalTestClassException("Client with clientId=\"" + annotation.client() + "\" not found in realm with ref=\"" + annotation.realmRef() + "\"")); adminBuilder.clientId(clientId).clientSecret(clientRep.getSecret()); if (userId != null) { UserRepresentation userRep = realmRep.getUsers().stream() .filter(u -> u.getUsername().equals(annotation.user())) - .findFirst().orElseThrow(() -> new TestFrameworkException("User " + annotation.user() + " not found in managed realm")); + .findFirst().orElseThrow(() -> new FatalTestClassException("User with username=\"" + annotation.user() + "\" not found in realm with ref=\"" + annotation.realmRef() + "\"")); String password = ManagedUser.getPassword(userRep); adminBuilder.username(userRep.getUsername()).password(password); adminBuilder.grantType(OAuth2Constants.PASSWORD); diff --git a/test-framework/core/src/main/java/org/keycloak/testframework/injection/Registry.java b/test-framework/core/src/main/java/org/keycloak/testframework/injection/Registry.java index 7f991946f4c..787d2ccf3cc 100644 --- a/test-framework/core/src/main/java/org/keycloak/testframework/injection/Registry.java +++ b/test-framework/core/src/main/java/org/keycloak/testframework/injection/Registry.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import org.keycloak.testframework.FatalTestClassException; import org.keycloak.testframework.TestFrameworkExecutor; import org.keycloak.testframework.annotations.TestCleanup; import org.keycloak.testframework.annotations.TestSetup; @@ -20,6 +21,7 @@ import org.keycloak.testframework.injection.predicates.RequestedInstancePredicat import org.keycloak.testframework.injection.predicates.TestFrameworkExecutorPredicates; import org.keycloak.testframework.server.KeycloakServer; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.InvocationInterceptor; import org.junit.jupiter.api.extension.ParameterContext; @@ -34,6 +36,7 @@ public class Registry implements AutoCloseable { private final Extensions extensions; private final List> deployedInstances = new LinkedList<>(); private final List> requestedInstances = new LinkedList<>(); + private FatalTestClassException fatalTestClassException; private Object currentTestInstance; @@ -118,18 +121,32 @@ public class Registry implements AutoCloseable { } public void beforeEach(Object testInstance, Method testMethod) { - findRequestedInstances(testInstance, testMethod); - destroyIncompatibleInstances(); - matchDeployedInstancesWithRequestedInstances(); - deployRequestedInstances(); - invokeBeforeEachOnSuppliers(); - injectFields(testInstance); - - if (currentTestInstance == null || testInstance.getClass() != currentTestInstance.getClass()) { - executeSetup(testInstance, TestSetup.class); + if (fatalTestClassException != null) { + skipTestMethod(); } - currentTestInstance = testInstance; + try { + findRequestedInstances(testInstance, testMethod); + destroyIncompatibleInstances(); + matchDeployedInstancesWithRequestedInstances(); + deployRequestedInstances(); + invokeBeforeEachOnSuppliers(); + injectFields(testInstance); + + if (currentTestInstance == null || testInstance.getClass() != currentTestInstance.getClass()) { + executeSetup(testInstance, TestSetup.class); + } + + currentTestInstance = testInstance; + } catch (FatalTestClassException e) { + requestedInstances.clear(); + fatalTestClassException = e; + skipTestMethod(); + } + } + + private void skipTestMethod() { + Assumptions.abort("Skipping test method due to fatal test class error"); } public void intercept(InvocationInterceptor.Invocation invocation, ReflectiveInvocationContext invocationContext) throws Throwable { @@ -262,11 +279,20 @@ public class Registry implements AutoCloseable { } public void afterAll() { - executeSetup(currentTestInstance, TestCleanup.class); + FatalTestClassException exception = fatalTestClassException; + fatalTestClassException = null; + + if (exception == null) { + executeSetup(currentTestInstance, TestCleanup.class); + } logger.logAfterAll(); List> destroy = deployedInstances.stream().filter(InstanceContextPredicates.hasLifeCycle(LifeCycle.CLASS)).toList(); destroy.forEach(this::destroy); + + if (exception != null) { + throw exception; + } } public void afterEach() { @@ -302,6 +328,9 @@ public class Registry implements AutoCloseable { for (Annotation annotation : annotations) { Supplier supplier = extensions.findSupplierByAnnotation(annotation); if (supplier != null) { + if (!supplier.getValueType().isAssignableFrom(valueType)) { + throw typeMismatch(annotation.annotationType(), supplier.getValueType(), valueType); + } return new RequestedInstance(supplier, annotation, valueType); } } @@ -381,6 +410,18 @@ public class Registry implements AutoCloseable { return extensions.getTestFrameworkExecutors().stream().filter(TestFrameworkExecutorPredicates.shouldExecute(testMethod)).findFirst().orElse(null); } + private FatalTestClassException typeMismatch( + Class annotation, + Class expectedType, + Class providedType) { + return new FatalTestClassException( + String.format("@%s requires %s (or its subclass) but field has type %s", + annotation.getSimpleName(), + expectedType.getName(), + providedType.getName()) + ); + } + private static class RequestedInstanceComparator implements Comparator { static final RequestedInstanceComparator INSTANCE = new RequestedInstanceComparator(); diff --git a/test-framework/core/src/test/java/org/keycloak/testframework/injection/RegistryTest.java b/test-framework/core/src/test/java/org/keycloak/testframework/injection/RegistryTest.java index 1f5de0750c7..45d1261ecae 100644 --- a/test-framework/core/src/test/java/org/keycloak/testframework/injection/RegistryTest.java +++ b/test-framework/core/src/test/java/org/keycloak/testframework/injection/RegistryTest.java @@ -3,6 +3,7 @@ package org.keycloak.testframework.injection; import java.lang.reflect.Method; import java.util.List; +import org.keycloak.testframework.FatalTestClassException; import org.keycloak.testframework.annotations.TestCleanup; import org.keycloak.testframework.annotations.TestSetup; import org.keycloak.testframework.config.Config; @@ -20,6 +21,7 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.opentest4j.TestAbortedException; public class RegistryTest { @@ -239,7 +241,7 @@ public class RegistryTest { } @Test - public void testMultiplRef() { + public void testMultipleRef() { MultipleRefTest refTest = new MultipleRefTest(); runBeforeEach(refTest); @@ -327,6 +329,21 @@ public class RegistryTest { Assertions.assertNotEquals(child1, test.child); } + @Test + public void testAnnotationValueTypeMismatch() { + AnnotationValueTypeMismatchTest test = new AnnotationValueTypeMismatchTest(); + + Assertions.assertThrows( + TestAbortedException.class, + () -> runBeforeEach(test) + ); + + Assertions.assertThrows( + FatalTestClassException.class, + () -> registry.afterAll() + ); + } + private void runBeforeEach(T testInstance) { try { Method testMethod = testInstance.getClass().getMethod("test"); @@ -419,7 +436,7 @@ public class RegistryTest { MockParentValue parent; @MockChildAnnotation - MockParentValue child; + MockChildValue child; @TestSetup public void setup() { @@ -440,4 +457,9 @@ public class RegistryTest { } + public static final class AnnotationValueTypeMismatchTest extends AbstractTest { + @MockParentAnnotation + MockChildValue child; + } + }