Merge pull request #134 from magento-commerce/MQE-2669_alex

MQE-2669: Seprated a run:failed command to generate:failed and run:fa…

Author: okolesnyk

dev/tests/unit/Magento/FunctionalTestFramework/Console/RunTestFailedCommandTest.php

@@ -0,0 +1,53 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace tests\unit\Magento\FunctionalTestFramework\Console;
+
+use Magento\FunctionalTestingFramework\Console\RunTestFailedCommand;
+
+class RunTestFailedCommandTest extends BaseGenerateCommandTest
+{
+    /**
+     * @throws \ReflectionException
+     */
+    public function testMultipleTests(): void
+    {
+        $testFailedFile = [
+            "tests/functional/tests/MFTF/_generated/default/SingleTestNoSuiteTest.php:SingleTestNoSuiteTest",
+            "tests/functional/tests/MFTF/_generated/SomeSpecificSuite/FirstTestSuiteTest.php:SingleTestSuiteTest",
+            "tests/functional/tests/MFTF/_generated/SomeSpecificSuite/SecondTestNoSuiteTest.php:SingleTestNoSuiteTest",
+            "tests/functional/tests/MFTF/_generated/SomeOtherSuite/SecondTestNoSuiteTest.php:SingleTestNoSuiteTest",
+        ];
+
+        $expectedResult = [
+            "tests/functional/tests/MFTF/_generated/default/SingleTestNoSuiteTest.php",
+            "-g SomeSpecificSuite",
+            "-g SomeOtherSuite",
+        ];
+
+        $runFailed = new RunTestFailedCommand('run:failed');
+        $filter = $this->invokePrivateMethod($runFailed, 'filterTestsForExecution', [$testFailedFile]);
+        $this->assertEquals($expectedResult, $filter);
+    }
+
+    /**
+     * Invoking private method to be able to test it.
+     * NOTE: Bad practice don't repeat it.
+     *
+     * @param $object
+     * @param $methodName
+     * @param array $parameters
+     * @return mixed
+     * @throws \ReflectionException
+     */
+    private function invokePrivateMethod(&$object, $methodName, array $parameters = [])
+    {
+        $reflection = new \ReflectionClass(get_class($object));
+        $method = $reflection->getMethod($methodName);
+        $method->setAccessible(true);
+        return $method->invokeArgs($object, $parameters);
+    }
+}

dev/tests/verification/Resources/ActionGroupContainsStepKeyInArgText.txt

@@ -41,4 +41,10 @@ class ActionGroupContainsStepKeyInArgTextCest
 		$I->see("arg1", ".selector"); // stepKey: arg1ActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupContainsStepKeyInArgValue");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupMergedViaInsertAfter.txt

@@ -35,4 +35,10 @@ class ActionGroupMergedViaInsertAfterCest
 		$I->fillField("#baz", "baz"); // stepKey: fillField3Keyone
 		$I->comment("Exiting Action Group [keyone] FunctionalActionGroupForMassMergeAfter");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupMergedViaInsertBefore.txt

@@ -35,4 +35,10 @@ class ActionGroupMergedViaInsertBeforeCest
 		$I->fillField("#baz", "baz"); // stepKey: fillField3Keyone
 		$I->comment("Exiting Action Group [keyone] FunctionalActionGroupForMassMergeBefore");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupReturningValueTest.txt

@@ -72,4 +72,10 @@ class ActionGroupReturningValueTestCest
 		$I->see($actionGroupWithReturnValue1, "#element .{$actionGroupWithReturnValue1}"); // stepKey: see1ActionGroupWithStringUsage1
 		$I->comment("Exiting Action Group [actionGroupWithStringUsage1] actionGroupWithStringUsage");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupToExtend.txt

@@ -31,4 +31,10 @@ class ActionGroupToExtendCest
 		$I->assertCount(99, $grabProductsActionGroup); // stepKey: assertCountActionGroup
 		$I->comment("Exiting Action Group [actionGroup] ActionGroupToExtend");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupUsingCreateData.txt

@@ -39,4 +39,10 @@ class ActionGroupUsingCreateDataCest
 	public function ActionGroupUsingCreateData(AcceptanceTester $I)
 	{
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupUsingNestedArgument.txt

@@ -31,4 +31,10 @@ class ActionGroupUsingNestedArgumentCest
 		$I->assertCount(99, $grabProductsActionGroup); // stepKey: assertCountActionGroup
 		$I->comment("Exiting Action Group [actionGroup] ActionGroupToExtend");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt

@@ -73,4 +73,10 @@ class ActionGroupWithDataOverrideTestCest
 		$I->comment("Exiting Action Group [actionGroupWithDataOverride1] FunctionalActionGroupWithData");
 		$I->click("loginButton"); // stepKey: step6
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithDataTest.txt

@@ -73,4 +73,10 @@ class ActionGroupWithDataTestCest
 		$I->comment("Exiting Action Group [actionGroupWithData1] FunctionalActionGroupWithData");
 		$I->click("loginButton"); // stepKey: step6
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithDefaultArgumentAndStringSelectorParam.txt

@@ -32,4 +32,10 @@ class ActionGroupWithDefaultArgumentAndStringSelectorParamCest
 		$I->see("John", "#element .test1"); // stepKey: seeFirstNameActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithDefaultArgumentAndStringSelectorParam");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithMultipleParameterSelectorsFromDefaultArgument.txt

@@ -32,4 +32,10 @@ class ActionGroupWithMultipleParameterSelectorsFromDefaultArgumentCest
 		$I->see("Doe", "#John-Doe .test"); // stepKey: seeLastNameActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithMultipleParameterSelectorsFromArgument");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithNoArguments.txt

@@ -32,4 +32,10 @@ class ActionGroupWithNoArgumentsCest
 		$I->wait(1); // stepKey: waitForNothingActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithoutArguments");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt

@@ -71,4 +71,10 @@ class ActionGroupWithNoDefaultTestCest
 		$I->comment("Exiting Action Group [actionGroupWithDataOverride1] FunctionalActionGroupNoDefault");
 		$I->click("loginButton"); // stepKey: step6
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithParameterizedElementWithHyphen.txt

@@ -30,4 +30,10 @@ class ActionGroupWithParameterizedElementWithHyphenCest
 		$keyoneActionGroup = $I->executeJS("#element .full-width"); // stepKey: keyoneActionGroup
 		$I->comment("Exiting Action Group [actionGroup] SectionArgumentWithParameterizedSelector");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithParameterizedElementsWithStepKeyReferences.txt

@@ -35,4 +35,10 @@ class ActionGroupWithParameterizedElementsWithStepKeyReferencesCest
 		$I->seeElement("//div[@name='Tiberius'][@class={$testVariableActionGroup}][@data-element='{$testVariable2ActionGroup}'][" . $I->retrieveEntityField('createSimpleData', 'name', 'test') . "]"); // stepKey: see1ActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithParametrizedSelectors");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithPassedArgumentAndStringSelectorParam.txt

@@ -32,4 +32,10 @@ class ActionGroupWithPassedArgumentAndStringSelectorParamCest
 		$I->see("John" . msq("UniquePerson"), "#element .test1"); // stepKey: seeFirstNameActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithDefaultArgumentAndStringSelectorParam");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithPersistedData.txt

@@ -72,4 +72,10 @@ class ActionGroupWithPersistedDataCest
 		$I->see("#element ." . $I->retrieveEntityField('createPerson', 'firstname', 'test')); // stepKey: see1ActionGroupWithPersistedData1
 		$I->comment("Exiting Action Group [actionGroupWithPersistedData1] FunctionalActionGroupWithData");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithSectionAndDataAsArguments.txt

@@ -30,4 +30,10 @@ class ActionGroupWithSectionAndDataAsArgumentsCest
 		$I->waitForElementVisible("#element .John", 10); // stepKey: arg1ActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithSectionAndData");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithSimpleDataUsageFromDefaultArgument.txt

@@ -32,4 +32,10 @@ class ActionGroupWithSimpleDataUsageFromDefaultArgumentCest
 		$I->see("stringLiteral", "#element .stringLiteral"); // stepKey: see1ActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithStringUsage");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithSimpleDataUsageFromPassedArgument.txt

@@ -59,4 +59,10 @@ class ActionGroupWithSimpleDataUsageFromPassedArgumentCest
 		$I->see($I->retrieveEntityField('simpleData', 'firstname[data_index]', 'test'), "#element ." . $I->retrieveEntityField('simpleData', 'firstname[data_index]', 'test')); // stepKey: see1ActionGroup7
 		$I->comment("Exiting Action Group [actionGroup7] actionGroupWithEntityUsage");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithSingleParameterSelectorFromDefaultArgument.txt

@@ -32,4 +32,10 @@ class ActionGroupWithSingleParameterSelectorFromDefaultArgumentCest
 		$I->see("Doe", "#element .John"); // stepKey: seeLastNameActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithSingleParameterSelectorFromArgument");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithSingleParameterSelectorFromPassedArgument.txt

@@ -32,4 +32,10 @@ class ActionGroupWithSingleParameterSelectorFromPassedArgumentCest
 		$I->see("Doe", "#element .John" . msq("UniquePerson")); // stepKey: seeLastNameActionGroup
 		$I->comment("Exiting Action Group [actionGroup] actionGroupWithSingleParameterSelectorFromArgument");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt

@@ -55,4 +55,10 @@ class ActionGroupWithStepKeyReferencesCest
 		$action16ActionGroup = $I->grabValueFrom($action16ActionGroup); // stepKey: action16ActionGroup
 		$I->comment("Exiting Action Group [actionGroup] FunctionActionGroupWithStepKeyReferences");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt

@@ -71,4 +71,10 @@ class ActionGroupWithTopLevelPersistedDataCest
 		$I->see("#element ." . $I->retrieveEntityField('createPersonParam', 'firstname', 'test')); // stepKey: see1ActionGroupWithPersistedData1
 		$I->comment("Exiting Action Group [actionGroupWithPersistedData1] FunctionalActionGroupWithData");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/ActionsInDifferentModulesSuite.txt

@@ -51,6 +51,8 @@ class ActionsInDifferentModulesSuite extends \Codeception\GroupObject
     private function executePreConditions()
     {
         if ($this->currentTestRun == 1) {
+            $this->testCount = $this->getTestCount();
+
             print sprintf(self::$HOOK_EXECUTION_INIT, "before");
 
             try {
@@ -215,4 +217,28 @@ class ActionsInDifferentModulesSuite extends \Codeception\GroupObject
         }
         return $module;
     }
+
+    /**
+     * Counts how many tests in group.
+     *
+     * @return integer
+     */
+    private function getTestCount()
+    {
+        $config = $this->getGlobalConfig();
+        if (empty($config['groups']) || empty($config['groups'][self::$group])) {
+            return $this->testCount;
+        }
+        $pathToGroupDir = TESTS_BP . DIRECTORY_SEPARATOR . array_first($config['groups'][self::$group]);
+        $pathToGroupCests = $pathToGroupDir . "*Cest.php";
+
+        $files = glob($pathToGroupCests);
+        if (is_array($files)) {
+            $qty = count($files);
+            print('In a group "' . self::$group . '" suite executor found ' . $qty . ' tests.' . PHP_EOL);
+            return $qty;
+        }
+
+        return $this->testCount;
+    }
 }

dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt

@@ -68,4 +68,10 @@ class ArgumentWithSameNameAsElementCest
 		$I->seeElement("#element .John"); // stepKey: see2ActionGroup1
 		$I->comment("Exiting Action Group [actionGroup1] FunctionalActionGroupWithTrickyArgument");
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/AssertTest.txt

@@ -160,4 +160,10 @@ class AssertTestCest
 		$I->assertNotRegExp('/placeholder\/thumbnail\.jpg/', $getSimpleProductThumbnail); // stepKey: simpleThumbnailIsNotDefault
 		$I->assertRegExp("#var\s+adminAnalyticsMetadata\s+=\s+{\s+(\"[\w_]+\":\s+\"[^\"]*?\",\s+)*?(\"[\w_]+\":\s+\"[^\"]*?\"\s+)};#s", $pageSource, "adminAnalyticsMetadata object is invalid"); // stepKey: validateadminAnalyticsMetadata
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/BasicActionGroupTest.txt

@@ -49,4 +49,10 @@ class BasicActionGroupTestCest
 		$I->comment("Exiting Action Group [actionGroup1] FunctionalActionGroup");
 		$I->click("loginButton"); // stepKey: step6
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }

dev/tests/verification/Resources/BasicFunctionalTest.txt

@@ -185,4 +185,10 @@ class BasicFunctionalTestCest
 		$I->waitForJS("someJsFunction", 30); // stepKey: waitForJSKey1
 		$I->waitForText("someInput", 30, ".functionalTestSelector"); // stepKey: waitForText1
 	}
+
+	public function _passed(AcceptanceTester $I)
+	{
+		// Deleting itself so that we can rerun only failed tests.
+		unlink(__FILE__);
+	}
 }