diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc index 1a90b20a5407..8ede33a6efa0 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc @@ -33,6 +33,7 @@ repository on GitHub. exception's _root_ cause matches all supplied conditions, for use with the `EngineTestKit`. * `ReflectionSupport` now supports scanning for classpath resources. +* Introduce `@BeforeSuite` and `@AfterSuite` annotations. [[release-notes-5.11.0-RC1-junit-jupiter]] diff --git a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc index d38a312d799f..ebb1d6495b04 100644 --- a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc +++ b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc @@ -48,3 +48,14 @@ include::{testDir}/example/SuiteDemo.java[tags=user_guide] NOTE: There are numerous configuration options for discovering and filtering tests in a test suite. Please consult the Javadoc of the `{suite-api-package}` package for a full list of supported annotations and further details. + +==== @BeforeSuite and @AfterSuite + +`@BeforeSuite` and `@AfterSuite` annotations can be used on methods inside a +`@Suite`-annotated class. They will be executed respectively before and after +all tests of the test suite. + +[source,java,indent=0] +---- +include::{testDir}/example/BeforeAndAfterSuiteDemo.java[tags=user_guide] +---- diff --git a/documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java b/documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java new file mode 100644 index 000000000000..c461417361bc --- /dev/null +++ b/documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java @@ -0,0 +1,34 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package example; + +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.api.SelectPackages; +import org.junit.platform.suite.api.Suite; + +//tag::user_guide[] +@Suite +@SelectPackages("example") +class BeforeAndAfterSuiteDemo { + + @BeforeSuite + static void beforeSuite() { + // executes before the test suite + } + + @AfterSuite + static void afterSuite() { + // executes after the test suite + } + +} +//end::user_guide[] diff --git a/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java new file mode 100644 index 000000000000..c2b5e33a3467 --- /dev/null +++ b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.api; + +import static org.apiguardian.api.API.Status.EXPERIMENTAL; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apiguardian.api.API; + +/** + * {@code @AfterSuite} is used to signal that the annotated method should be + * executed after all tests in the current test suite. + * + *

Method Signatures

+ * + *

{@code @AfterSuite} methods must have a {@code void} return type, must + * be {@code static} and must not be {@code private}. + * + *

Inheritance and Execution Order

+ * + *

{@code @AfterSuite} methods are inherited from superclasses as long as they + * are not overridden according to the visibility rules of the Java + * language. Furthermore, {@code @AfterSuite} methods from superclasses will be + * executed after {@code @AfterSuite} methods in subclasses. + * + *

The JUnit Platform Suite Engine does not guarantee the execution order of + * multiple {@code @AfterSuite} methods that are declared within a single test + * class or test interface. While it may at times appear that these methods are + * invoked in alphabetical order, they are in fact sorted using an algorithm + * that is deterministic but intentionally non-obvious. + * + *

In addition, {@code @AfterSuite} methods are in no way linked to + * {@code @BeforeSuite} methods. Consequently, there are no guarantees with regard + * to their wrapping behavior. For example, given two + * {@code @BeforeSuite} methods {@code createA()} and {@code createB()} as well as + * two {@code @AfterSuite} methods {@code destroyA()} and {@code destroyB()}, the + * order in which the {@code @BeforeSuite} methods are executed (e.g. + * {@code createA()} before {@code createB()}) does not imply any order for the + * seemingly corresponding {@code @AfterSuite} methods. In other words, + * {@code destroyA()} might be called before or after + * {@code destroyB()}. The JUnit Team therefore recommends that developers + * declare at most one {@code @BeforeSuite} method and at most one + * {@code @AfterSuite} method per test class or test interface unless there are no + * dependencies between the {@code @BeforeSuite} methods or between the + * {@code @AfterSuite} methods. + * + *

Composition

+ * + *

{@code @AfterSuite} may be used as a meta-annotation in order to create + * a custom composed annotation that inherits the semantics of + * {@code @AfterSuite}. + * + * @since 1.11 + * @see BeforeSuite + * @see Suite + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@API(status = EXPERIMENTAL, since = "1.11") +public @interface AfterSuite { +} diff --git a/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java new file mode 100644 index 000000000000..5b4c0d3c93f0 --- /dev/null +++ b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.api; + +import static org.apiguardian.api.API.Status.EXPERIMENTAL; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apiguardian.api.API; + +/** + * {@code @BeforeSuite} is used to signal that the annotated method should be + * executed before all tests in the current test suite. + * + *

Method Signatures

+ * + *

{@code @BeforeSuite} methods must have a {@code void} return type, must + * be {@code static} and must not be {@code private}. + * + *

Inheritance and Execution Order

+ * + *

{@code @BeforeSuite} methods are inherited from superclasses as long as they + * are not overridden according to the visibility rules of the Java + * language. Furthermore, {@code @BeforeSuite} methods from superclasses will be + * executed before {@code @BeforeSuite} methods in subclasses. + * + *

The JUnit Platform Suite Engine does not guarantee the execution order of + * multiple {@code @BeforeSuite} methods that are declared within a single test + * class or test interface. While it may at times appear that these methods are + * invoked in alphabetical order, they are in fact sorted using an algorithm + * that is deterministic but intentionally non-obvious. + * + *

In addition, {@code @BeforeSuite} methods are in no way linked to + * {@code @AfterSuite} methods. Consequently, there are no guarantees with regard + * to their wrapping behavior. For example, given two + * {@code @BeforeSuite} methods {@code createA()} and {@code createB()} as well as + * two {@code @AfterSuite} methods {@code destroyA()} and {@code destroyB()}, the + * order in which the {@code @BeforeSuite} methods are executed (e.g. + * {@code createA()} before {@code createB()}) does not imply any order for the + * seemingly corresponding {@code @AfterSuite} methods. In other words, + * {@code destroyA()} might be called before or after + * {@code destroyB()}. The JUnit Team therefore recommends that developers + * declare at most one {@code @BeforeSuite} method and at most one + * {@code @AfterSuite} method per test class or test interface unless there are no + * dependencies between the {@code @BeforeSuite} methods or between the + * {@code @AfterSuite} methods. + * + *

Composition

+ * + *

{@code @BeforeSuite} may be used as a meta-annotation in order to create + * a custom composed annotation that inherits the semantics of + * {@code @BeforeSuite}. + * + * @since 1.11 + * @see AfterSuite + * @see Suite + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@API(status = EXPERIMENTAL, since = "1.11") +public @interface BeforeSuite { +} diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java new file mode 100644 index 000000000000..b00bca64a131 --- /dev/null +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine; + +import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedMethods; +import static org.junit.platform.commons.util.ReflectionUtils.returnsPrimitiveVoid; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.List; + +import org.junit.platform.commons.JUnitException; +import org.junit.platform.commons.util.ReflectionUtils; +import org.junit.platform.commons.util.ReflectionUtils.HierarchyTraversalMode; +import org.junit.platform.engine.support.hierarchical.ThrowableCollector; +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; + +/** + * Collection of utilities for working with test lifecycle methods. + * + * @since 1.11 + */ +final class LifecycleMethodUtils { + + private LifecycleMethodUtils() { + /* no-op */ + } + + static List findBeforeSuiteMethods(Class testClass, ThrowableCollector throwableCollector) { + return findMethodsAndAssertStaticAndNonPrivate(testClass, BeforeSuite.class, HierarchyTraversalMode.TOP_DOWN, + throwableCollector); + } + + static List findAfterSuiteMethods(Class testClass, ThrowableCollector throwableCollector) { + return findMethodsAndAssertStaticAndNonPrivate(testClass, AfterSuite.class, HierarchyTraversalMode.BOTTOM_UP, + throwableCollector); + } + + private static List findMethodsAndAssertStaticAndNonPrivate(Class testClass, + Class annotationType, HierarchyTraversalMode traversalMode, + ThrowableCollector throwableCollector) { + + List methods = findAnnotatedMethods(testClass, annotationType, traversalMode); + throwableCollector.execute(() -> methods.forEach(method -> { + assertVoid(annotationType, method); + assertStatic(annotationType, method); + assertNonPrivate(annotationType, method); + assertNoParameters(annotationType, method); + })); + return methods; + } + + private static void assertStatic(Class annotationType, Method method) { + if (ReflectionUtils.isNotStatic(method)) { + throw new JUnitException(String.format("@%s method '%s' must be static.", annotationType.getSimpleName(), + method.toGenericString())); + } + } + + private static void assertNonPrivate(Class annotationType, Method method) { + if (ReflectionUtils.isPrivate(method)) { + throw new JUnitException(String.format("@%s method '%s' must not be private.", + annotationType.getSimpleName(), method.toGenericString())); + } + } + + private static void assertVoid(Class annotationType, Method method) { + if (!returnsPrimitiveVoid(method)) { + throw new JUnitException(String.format("@%s method '%s' must not return a value.", + annotationType.getSimpleName(), method.toGenericString())); + } + } + + private static void assertNoParameters(Class annotationType, Method method) { + if (method.getParameterCount() > 0) { + throw new JUnitException(String.format("@%s method '%s' must not accept parameters.", + annotationType.getSimpleName(), method.toGenericString())); + } + } + +} diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java index 9fa107cb5f2c..ccf40405f7bd 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java @@ -13,8 +13,12 @@ import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation; import static org.junit.platform.suite.commons.SuiteLauncherDiscoveryRequestBuilder.request; +import java.lang.reflect.Method; +import java.util.List; + import org.junit.platform.commons.JUnitException; import org.junit.platform.commons.util.Preconditions; +import org.junit.platform.commons.util.ReflectionUtils; import org.junit.platform.commons.util.StringUtils; import org.junit.platform.engine.ConfigurationParameters; import org.junit.platform.engine.EngineExecutionListener; @@ -24,6 +28,8 @@ import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector; +import org.junit.platform.engine.support.hierarchical.ThrowableCollector; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.core.LauncherDiscoveryResult; import org.junit.platform.launcher.listeners.TestExecutionSummary; @@ -121,16 +127,58 @@ private static String getSuiteDisplayName(Class testClass) { void execute(EngineExecutionListener parentEngineExecutionListener) { parentEngineExecutionListener.executionStarted(this); + ThrowableCollector throwableCollector = new OpenTest4JAwareThrowableCollector(); + + List beforeSuiteMethods = LifecycleMethodUtils.findBeforeSuiteMethods(suiteClass, throwableCollector); + List afterSuiteMethods = LifecycleMethodUtils.findAfterSuiteMethods(suiteClass, throwableCollector); + + executeBeforeSuiteMethods(beforeSuiteMethods, throwableCollector); + + TestExecutionSummary summary = executeTests(parentEngineExecutionListener, throwableCollector); + + executeAfterSuiteMethods(afterSuiteMethods, throwableCollector); + + TestExecutionResult testExecutionResult = computeTestExecutionResult(summary, throwableCollector); + parentEngineExecutionListener.executionFinished(this, testExecutionResult); + } + + private void executeBeforeSuiteMethods(List beforeSuiteMethods, ThrowableCollector throwableCollector) { + if (throwableCollector.isNotEmpty()) { + return; + } + for (Method beforeSuiteMethod : beforeSuiteMethods) { + throwableCollector.execute(() -> ReflectionUtils.invokeMethod(beforeSuiteMethod, null)); + if (throwableCollector.isNotEmpty()) { + return; + } + } + } + + private TestExecutionSummary executeTests(EngineExecutionListener parentEngineExecutionListener, + ThrowableCollector throwableCollector) { + if (throwableCollector.isNotEmpty()) { + return null; + } + // #2838: The discovery result from a suite may have been filtered by // post discovery filters from the launcher. The discovery result should // be pruned accordingly. LauncherDiscoveryResult discoveryResult = this.launcherDiscoveryResult.withRetainedEngines( getChildren()::contains); - TestExecutionSummary summary = launcher.execute(discoveryResult, parentEngineExecutionListener); - parentEngineExecutionListener.executionFinished(this, computeTestExecutionResult(summary)); + return launcher.execute(discoveryResult, parentEngineExecutionListener); } - private TestExecutionResult computeTestExecutionResult(TestExecutionSummary summary) { + private void executeAfterSuiteMethods(List afterSuiteMethods, ThrowableCollector throwableCollector) { + for (Method afterSuiteMethod : afterSuiteMethods) { + throwableCollector.execute(() -> ReflectionUtils.invokeMethod(afterSuiteMethod, null)); + } + } + + private TestExecutionResult computeTestExecutionResult(TestExecutionSummary summary, + ThrowableCollector throwableCollector) { + if (throwableCollector.isNotEmpty()) { + return TestExecutionResult.failed(throwableCollector.getThrowable()); + } if (failIfNoTests && summary.getTestsFoundCount() == 0) { return TestExecutionResult.failed(new NoTestsDiscoveredException(suiteClass)); } diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java new file mode 100644 index 000000000000..c892eee146bb --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.suite.engine.SuiteEngineDescriptor.ENGINE_ID; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingBeforeAndAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonStaticAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonStaticBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonVoidAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonVoidBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.ParameterAcceptingAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.ParameterAcceptingBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.PrivateAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.PrivateBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.SeveralFailingBeforeAndAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.SubclassWithBeforeAndAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.SuccessfulBeforeAndAfterSuite; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.suppressed; + +import java.util.ArrayList; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.platform.commons.JUnitException; +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.engine.testcases.StatefulTestCase; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; + +/** + * Integration tests that verify support for {@link BeforeSuite} and {@link AfterSuite}, + * in the {@link SuiteTestEngine}. + * + * @since 1.11 + */ +public class BeforeAndAfterSuiteTests { + + @BeforeEach + void setUp() { + StatefulTestCase.callSequence = new ArrayList<>(); + } + + @Test + void successfulBeforeAndAfterSuite() { + // @formatter:off + executeSuite(SuccessfulBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(7).finished(7).succeeded(6).failed(1)) + .assertThatEvents() + .haveExactly(1, event(test(StatefulTestCase.Test1.class.getName()), finishedSuccessfully())) + .haveExactly(1, event(test(StatefulTestCase.Test2.class.getName()), finishedWithFailure())); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "test1", + "test2", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void beforeAndAfterSuiteInheritance() { + // @formatter:off + executeSuite(SubclassWithBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(7).finished(7).succeeded(6).failed(1)); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "superclassBeforeSuiteMethod", + "subclassBeforeSuiteMethod", + "test1", + "test2", + "subclassAfterSuiteMethod", + "superclassAfterSuiteMethod" + ); + // @formatter:on + } + + @Test + void failingBeforeSuite() { + // @formatter:off + executeSuite(FailingBeforeSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(2).finished(2).succeeded(1).failed(1)) + .assertThatEvents() + .haveExactly(1, event( + container(FailingBeforeSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @BeforeSuite method")))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void failingAfterSuite() { + // @formatter:off + executeSuite(FailingAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(7).finished(7).succeeded(5).failed(2)) + .assertThatEvents() + .haveExactly(1, event( + container(FailingAfterSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method")))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "test1", + "test2", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void failingBeforeAndAfterSuite() { + // @formatter:off + executeSuite(FailingBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(2).finished(2).succeeded(1).failed(1)) + .assertThatEvents() + .haveExactly(1, event( + container(FailingBeforeAndAfterSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @BeforeSuite method"), + suppressed(0, instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method"))))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void severalFailingBeforeAndAfterSuite() { + // @formatter:off + executeSuite(SeveralFailingBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(2).finished(2).succeeded(1).failed(1)) + .assertThatEvents() + .haveExactly(1, event( + container(SeveralFailingBeforeAndAfterSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @BeforeSuite method"), + suppressed(0, instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method")), + suppressed(1, instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method"))))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "afterSuiteMethod", + "afterSuiteMethod" + ); + // @formatter:on + } + + @ParameterizedTest(name = "{0}") + @MethodSource + void invalidBeforeOrAfterSuiteMethod(Class testSuiteClass, Predicate failureMessagePredicate) { + // @formatter:off + executeSuite(testSuiteClass) + .allEvents() + .assertThatEvents() + .haveExactly(1, event( + container(testSuiteClass), + finishedWithFailure(instanceOf(JUnitException.class), message(failureMessagePredicate)))); + // @formatter:on + } + + private static Stream invalidBeforeOrAfterSuiteMethod() { + return Stream.of( + invalidBeforeOrAfterSuiteCase(NonVoidBeforeSuite.class, "@BeforeSuite method", "must not return a value."), + invalidBeforeOrAfterSuiteCase(ParameterAcceptingBeforeSuite.class, "@BeforeSuite method", + "must not accept parameters."), + invalidBeforeOrAfterSuiteCase(NonStaticBeforeSuite.class, "@BeforeSuite method", "must be static."), + invalidBeforeOrAfterSuiteCase(PrivateBeforeSuite.class, "@BeforeSuite method", "must not be private."), + invalidBeforeOrAfterSuiteCase(NonVoidAfterSuite.class, "@AfterSuite method", "must not return a value."), + invalidBeforeOrAfterSuiteCase(ParameterAcceptingAfterSuite.class, "@AfterSuite method", + "must not accept parameters."), + invalidBeforeOrAfterSuiteCase(NonStaticAfterSuite.class, "@AfterSuite method", "must be static."), + invalidBeforeOrAfterSuiteCase(PrivateAfterSuite.class, "@AfterSuite method", "must not be private.")); + } + + private static Arguments invalidBeforeOrAfterSuiteCase(Class suiteClass, String failureMessageStart, + String failureMessageEnd) { + return arguments(named(suiteClass.getSimpleName(), suiteClass), + (Predicate) s -> s.startsWith(failureMessageStart) && s.endsWith(failureMessageEnd)); + } + + private static EngineExecutionResults executeSuite(Class suiteClass) { + return EngineTestKit.engine(ENGINE_ID).selectors(selectClass(suiteClass)).execute(); + } + +} diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java new file mode 100644 index 000000000000..3a2a32230c27 --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine.testcases; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * @since 1.11 + */ +public class StatefulTestCase { + + public static List callSequence = new ArrayList<>(); + + public static class Test1 { + + @Test + void statefulTest() { + callSequence.add("test1"); + } + + } + + public static class Test2 { + + @Test + void statefulTest() { + callSequence.add("test2"); + fail("This is a failing test"); + } + + } + +} diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java new file mode 100644 index 000000000000..8a5af3587d38 --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java @@ -0,0 +1,245 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine.testsuites; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; +import org.junit.platform.suite.engine.BeforeAndAfterSuiteTests; +import org.junit.platform.suite.engine.testcases.StatefulTestCase; + +/** + * Test suites used in {@link BeforeAndAfterSuiteTests}. + * + * @since 1.11 + */ +public class LifecycleMethodsSuites { + + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + @Suite + @SelectClasses({ StatefulTestCase.Test1.class, StatefulTestCase.Test2.class }) + private @interface TestSuite { + } + + @TestSuite + public static class SuccessfulBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + } + + } + + @TestSuite + public static class FailingBeforeSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + } + + } + + @TestSuite + public static class FailingAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + } + + @TestSuite + public static class FailingBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + } + + @TestSuite + public static class SeveralFailingBeforeAndAfterSuite { + + @BeforeSuite + static void setUp1() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @BeforeSuite + static void setUp2() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @AfterSuite + static void tearDown1() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + @AfterSuite + static void tearDown2() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + } + + @TestSuite + public static class SuperclassWithBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("superclassBeforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("superclassAfterSuiteMethod"); + } + + } + + public static class SubclassWithBeforeAndAfterSuite extends SuperclassWithBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("subclassBeforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("subclassAfterSuiteMethod"); + } + + } + + @TestSuite + public static class NonVoidBeforeSuite { + + @BeforeSuite + static String nonVoidBeforeSuite() { + fail("Should not be called"); + return ""; + } + + } + + @TestSuite + public static class ParameterAcceptingBeforeSuite { + + @BeforeSuite + static void parameterAcceptingBeforeSuite(String param) { + fail("Should not be called"); + } + + } + + @TestSuite + public static class NonStaticBeforeSuite { + + @BeforeSuite + void nonStaticBeforeSuite() { + fail("Should not be called"); + } + + } + + @TestSuite + public static class PrivateBeforeSuite { + + @BeforeSuite + private static void privateBeforeSuite() { + fail("Should not be called"); + } + + } + + @TestSuite + public static class NonVoidAfterSuite { + + @AfterSuite + static String nonVoidAfterSuite() { + fail("Should not be called"); + return ""; + } + + } + + @TestSuite + public static class ParameterAcceptingAfterSuite { + + @AfterSuite + static void parameterAcceptingAfterSuite(String param) { + fail("Should not be called"); + } + + } + + @TestSuite + public static class NonStaticAfterSuite { + + @AfterSuite + void nonStaticAfterSuite() { + fail("Should not be called"); + } + + } + + @TestSuite + public static class PrivateAfterSuite { + + @AfterSuite + private static void privateAfterSuite() { + fail("Should not be called"); + } + + } + +}