Parameter alwaysRun=true for before-methods forces execution of those methods

Closes testng-team#1622

Author: krmahadevan

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB-1622: Parameter alwaysRun=true for before-methods forces execution of those methods (Krishnan Mahadevan)
 Fixed: GITHUB-1893: Streamline invocation of "init" method within TestResult to be private (Krishnan Mahadevan)
 Fixed: GITHUB-1892: Configurable InvokedMethodListener (Krishnan Mahadevan)
 Fixed: GITHUB-435: Apply <packages> at suite level to all tests (Siegmar Alber)

src/main/java/org/testng/internal/Invoker.java

@@ -201,7 +201,10 @@ private void invokeConfigurations(
           log(3, "Skipping " + Utils.detailedMethodName(tm, true) + " because it is not enabled");
           continue;
         }
-        if (!confInvocationPassed(tm, currentTestMethod, testClass, instance) && !alwaysRun) {
+        boolean considerFailures =
+                m_testContext.getSuite().getXmlSuite().getConfigFailurePolicy() != XmlSuite.FailurePolicy.CONTINUE;
+        boolean condition = considerFailures || !alwaysRun;
+        if (!confInvocationPassed(tm, currentTestMethod, testClass, instance) && condition) {
           log(3, "Skipping " + Utils.detailedMethodName(tm, true));
           InvokedMethod invokedMethod =
               new InvokedMethod(instance, tm, System.currentTimeMillis(), testResult);

src/test/java/test/alwaysrun/AlwaysRunTest.java

@@ -6,6 +6,7 @@
 import org.testng.TestNG;
 import org.testng.annotations.Test;
 
+import org.testng.xml.XmlSuite;
 import test.SimpleBaseTest;
 import testhelper.OutputDirectoryPatch;
 
@@ -18,6 +19,7 @@ public void withAlwaysRunAfter() {
     testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
     testng.setTestClasses(new Class[] {AlwaysRunAfter1.class});
     testng.addListener(tla);
+    testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
     testng.run();
     assertTrue(AlwaysRunAfter1.success(), "afterTestMethod should have run");
   }
@@ -29,6 +31,7 @@ public void withAlwaysRunAfterMethod() {
     testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
     testng.setTestClasses(new Class[] {AlwaysRunAfter3.class});
     testng.addListener(tla);
+    testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
     testng.run();
     assertTrue(AlwaysRunAfter3.success(), "afterMethod should have run");
   }

src/test/java/test/configuration/github1622/Issue1622Test.java

@@ -0,0 +1,30 @@
+package test.configuration.github1622;
+
+import org.assertj.core.api.Assertions;
+import org.testng.ITestNGListener;
+import org.testng.TestNG;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import org.testng.xml.XmlSuite;
+import test.InvokedMethodNameListener;
+import test.SimpleBaseTest;
+
+public class Issue1622Test extends SimpleBaseTest {
+    @Test(dataProvider = "dp")
+    public void testWithoutGroups(XmlSuite.FailurePolicy failurePolicy, String ...expected) {
+        TestNG testNG = create(TestClassWithNoGroups.class);
+        testNG.setConfigFailurePolicy(failurePolicy);
+        InvokedMethodNameListener listener = new InvokedMethodNameListener();
+        testNG.addListener((ITestNGListener) listener);
+        testNG.run();
+        Assertions.assertThat(listener.getInvokedMethodNames()).contains(expected);
+    }
+
+    @DataProvider(name = "dp")
+    public Object[][] getData() {
+        return new Object[][] {
+                {XmlSuite.FailurePolicy.SKIP, "failedBeforeSuite"},
+                {XmlSuite.FailurePolicy.CONTINUE, "failedBeforeSuite", "beforeTest", "beforeClass", "beforeMethod"}
+        };
+    }
+}

src/test/java/test/configuration/github1622/TestClassWithNoGroups.java

@@ -0,0 +1,41 @@
+package test.configuration.github1622;
+
+import org.testng.ITestNGListener;
+import org.testng.TestNG;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+import org.testng.xml.XmlSuite;
+import test.InvokedMethodNameListener;
+import test.SimpleBaseTest;
+
+public class TestClassWithNoGroups {
+
+    @BeforeSuite
+    public void failedBeforeSuite() {
+        throw new RuntimeException();
+    }
+
+    @BeforeTest(alwaysRun = true)
+    public void beforeTest() {
+        throw new RuntimeException();
+    }
+
+    @BeforeClass(alwaysRun = true)
+    public void beforeClass() {
+        throw new RuntimeException();
+    }
+
+    @BeforeMethod(alwaysRun = true)
+    public void beforeMethod() {
+        throw new RuntimeException();
+    }
+
+    @Test
+    public void testMethod() {
+        System.out.println("I'm testMethod");
+    }
+}

src/test/resources/testng.xml

@@ -183,7 +183,7 @@
       <class name="test.factory.github328.GitHub328Test" />
       <class name="test.factory.issue1745.Github1745Test"/>
       <class name="test.objectfactory.github1827.GitHub1827Test"/>
-      <class name="test.github1490.VerifyDataProviderListener"/>      
+      <class name="test.github1490.VerifyDataProviderListener"/>
       <class name="test.methodselection.MethodSelectionTest"/>
       <class name="test.beforegroups.BeforeGroupsTest"/>
       <class name="test.aftergroups.issue1880.IssueTest"/>
@@ -785,7 +785,7 @@
       <class name="test.sanitycheck.CheckSuiteNamesTest" />
     </classes>
   </test>
-  
+
   <test name="TestNames">
     <classes>
       <class name="org.testng.xml.internal.TestNamesMatcherTest"/>
@@ -812,6 +812,7 @@
       <class name="test.configuration.BeforeClassWithDisabledTest" />
       <class name="test.configuration.github1625.TestRunnerIssue1625"/>
       <class name="test.configuration.issue1753.IssueTest"/>
+      <class name="test.configuration.github1622.Issue1622Test"/>
     </classes>
   </test>