Skip to content

Commit 3343e8e

Browse files
santysisisantysisi
committed
add Retry attribute to support retrying flaky test
1 parent 081b08e commit 3343e8e

File tree

12 files changed

+503
-1
lines changed

12 files changed

+503
-1
lines changed

‎src/Framework/Attributes/Retry.php

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
<?php declare(strict_types=1);
2+
/*
3+
* This file is part of PHPUnit.
4+
*
5+
* (c) Sebastian Bergmann <[email protected]>
6+
*
7+
* For the full copyright and license information, please view the LICENSE
8+
* file that was distributed with this source code.
9+
*/
10+
namespace PHPUnit\Framework\Attributes;
11+
12+
use Attribute;
13+
14+
/**
15+
* @immutable
16+
*
17+
* @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit
18+
*/
19+
#[Attribute(Attribute::TARGET_METHOD | Attribute::IS_REPEATABLE)]
20+
final readonly class Retry
21+
{
22+
private int $maxRetries;
23+
private int $delay;
24+
25+
/**
26+
* @var ?non-empty-string
27+
*/
28+
private ?string $retryOn;
29+
30+
/**
31+
* @param ?non-empty-string $retryOn
32+
*/
33+
public function __construct(int $maxRetries, ?int $delay = 0, ?string $retryOn = null)
34+
{
35+
$this->maxRetries = $maxRetries;
36+
$this->delay = $delay;
37+
$this->retryOn = $retryOn;
38+
}
39+
40+
public function maxRetries(): int
41+
{
42+
return $this->maxRetries;
43+
}
44+
45+
public function delay(): int
46+
{
47+
return $this->delay;
48+
}
49+
50+
/**
51+
* @return ?non-empty-string
52+
*/
53+
public function retryOn(): ?string
54+
{
55+
return $this->retryOn;
56+
}
57+
}

‎src/Framework/TestCase.php

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
use function restore_exception_handler;
4747
use function set_error_handler;
4848
use function set_exception_handler;
49+
use function sleep;
4950
use function sprintf;
5051
use function str_contains;
5152
use function stream_get_contents;
@@ -79,6 +80,7 @@
7980
use PHPUnit\Metadata\Api\HookMethods;
8081
use PHPUnit\Metadata\Api\Requirements;
8182
use PHPUnit\Metadata\Parser\Registry as MetadataRegistry;
83+
use PHPUnit\Metadata\Retry;
8284
use PHPUnit\Metadata\WithEnvironmentVariable;
8385
use PHPUnit\Runner\BackedUpEnvironmentVariable;
8486
use PHPUnit\Runner\DeprecationCollector\Facade as DeprecationCollector;
@@ -1248,7 +1250,7 @@ protected function onNotSuccessfulTest(Throwable $t): never
12481250
* @throws ExpectationFailedException
12491251
* @throws Throwable
12501252
*/
1251-
private function runTest(): mixed
1253+
private function runTest(int $attempt = 0): mixed
12521254
{
12531255
$testArguments = array_merge($this->data, array_values($this->dependencyInput));
12541256

@@ -1276,6 +1278,12 @@ private function runTest(): mixed
12761278
}
12771279

12781280
if (!$this->shouldExceptionExpectationsBeVerified($exception)) {
1281+
$metadata = $this->getRetryMetadata($exception, $attempt);
1282+
1283+
if (null !== $metadata) {
1284+
return $this->retryTest($metadata, $attempt);
1285+
}
1286+
12791287
throw $exception;
12801288
}
12811289

@@ -2255,6 +2263,32 @@ private function handleExceptionFromInvokedCountMockObjectRule(Throwable $t): vo
22552263
}
22562264
}
22572265

2266+
private function getRetryMetadata(Throwable $th, int $attempt): ?Retry
2267+
{
2268+
if (MetadataRegistry::parser()->forMethod($this::class, $this->name())->isRetry()->isEmpty()) {
2269+
return null;
2270+
}
2271+
2272+
foreach (MetadataRegistry::parser()->forMethod($this::class, $this->name())->isRetry() as $metadata) {
2273+
assert($metadata instanceof Retry);
2274+
2275+
if ($metadata->maxRetries() > $attempt && (null === $metadata->retryOn() || $th instanceof ($metadata->retryOn()))) {
2276+
return $metadata;
2277+
}
2278+
}
2279+
2280+
return null;
2281+
}
2282+
2283+
private function retryTest(Retry $metadata, int $attempt): mixed
2284+
{
2285+
if ($metadata->delay() > 0) {
2286+
sleep($metadata->delay());
2287+
}
2288+
2289+
return $this->runTest(++$attempt);
2290+
}
2291+
22582292
/**
22592293
* Creates a test stub for the specified interface or class.
22602294
*

‎src/Metadata/Metadata.php

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,11 @@ public static function withoutErrorHandler(): WithoutErrorHandler
534534
return new WithoutErrorHandler(self::METHOD_LEVEL);
535535
}
536536

537+
public static function retry(int $maxRetries, ?int $delay = 0, ?string $retryOn = null): Retry
538+
{
539+
return new Retry(self::METHOD_LEVEL, $maxRetries, $delay, $retryOn);
540+
}
541+
537542
/**
538543
* @param int<0, 1> $level
539544
*/
@@ -969,4 +974,12 @@ public function isWithoutErrorHandler(): bool
969974
{
970975
return false;
971976
}
977+
978+
/**
979+
* @phpstan-assert-if-true Retry $this
980+
*/
981+
public function isRetry(): bool
982+
{
983+
return false;
984+
}
972985
}

‎src/Metadata/MetadataCollection.php

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -632,4 +632,14 @@ public function isWithoutErrorHandler(): self
632632
),
633633
);
634634
}
635+
636+
public function isRetry(): self
637+
{
638+
return new self(
639+
...array_filter(
640+
$this->metadata,
641+
static fn (Metadata $metadata): bool => $metadata->isRetry(),
642+
),
643+
);
644+
}
635645
}

‎src/Metadata/Parser/AttributeParser.php

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
use PHPUnit\Framework\Attributes\RequiresPhpunit;
6868
use PHPUnit\Framework\Attributes\RequiresPhpunitExtension;
6969
use PHPUnit\Framework\Attributes\RequiresSetting;
70+
use PHPUnit\Framework\Attributes\Retry;
7071
use PHPUnit\Framework\Attributes\RunClassInSeparateProcess;
7172
use PHPUnit\Framework\Attributes\RunInSeparateProcess;
7273
use PHPUnit\Framework\Attributes\RunTestsInSeparateProcesses;
@@ -845,6 +846,13 @@ public function forMethod(string $className, string $methodName): MetadataCollec
845846

846847
$result[] = Metadata::withoutErrorHandler();
847848

849+
break;
850+
851+
case Retry::class:
852+
assert($attributeInstance instanceof Retry);
853+
854+
$result[] = Metadata::retry($attributeInstance->maxRetries(), $attributeInstance->delay(), $attributeInstance->retryOn());
855+
848856
break;
849857
}
850858
}

‎src/Metadata/Retry.php

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
<?php declare(strict_types=1);
2+
/*
3+
* This file is part of PHPUnit.
4+
*
5+
* (c) Sebastian Bergmann <[email protected]>
6+
*
7+
* For the full copyright and license information, please view the LICENSE
8+
* file that was distributed with this source code.
9+
*/
10+
namespace PHPUnit\Metadata;
11+
12+
/**
13+
* @immutable
14+
*
15+
* @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit
16+
*/
17+
final readonly class Retry extends Metadata
18+
{
19+
private int $maxRetries;
20+
private int $delay;
21+
22+
/**
23+
* @var ?non-empty-string
24+
*/
25+
private ?string $retryOn;
26+
27+
/**
28+
* @param ?non-empty-string $retryOn
29+
*/
30+
public function __construct(int $level, int $maxRetries, int $delay, ?string $retryOn)
31+
{
32+
parent::__construct($level);
33+
34+
$this->maxRetries = $maxRetries;
35+
$this->delay = $delay;
36+
$this->retryOn = $retryOn;
37+
}
38+
39+
public function isRetry(): bool
40+
{
41+
return true;
42+
}
43+
44+
public function maxRetries(): int
45+
{
46+
return $this->maxRetries;
47+
}
48+
49+
public function delay(): int
50+
{
51+
return $this->delay;
52+
}
53+
54+
/**
55+
* @return ?non-empty-string
56+
*/
57+
public function retryOn(): ?string
58+
{
59+
return $this->retryOn;
60+
}
61+
}

‎tests/_files/Metadata/Attribute/tests/RetryTest.php

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?php declare(strict_types=1);
2+
/*
3+
* This file is part of PHPUnit.
4+
*
5+
* (c) Sebastian Bergmann <[email protected]>
6+
*
7+
* For the full copyright and license information, please view the LICENSE
8+
* file that was distributed with this source code.
9+
*/
10+
namespace PHPUnit\TestFixture\Metadata\Attribute;
11+
12+
use LogicException;
13+
use PHPUnit\Framework\Attributes\Retry;
14+
use PHPUnit\Framework\TestCase;
15+
16+
final class RetryTest extends TestCase
17+
{
18+
#[Retry(1)]
19+
#[Retry(2, 1)]
20+
#[Retry(3, 0, LogicException::class)]
21+
public function testOne(): void
22+
{
23+
}
24+
}

‎tests/unit/Framework/Attributes/RetryTest.php

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
<?php declare(strict_types=1);
2+
/*
3+
* This file is part of PHPUnit.
4+
*
5+
* (c) Sebastian Bergmann <[email protected]>
6+
*
7+
* For the full copyright and license information, please view the LICENSE
8+
* file that was distributed with this source code.
9+
*/
10+
namespace PHPUnit\Framework\Attributes;
11+
12+
use DateTime;
13+
use Exception;
14+
use LogicException;
15+
use PHPUnit\Framework\TestCase;
16+
17+
final class RetryTest extends TestCase
18+
{
19+
private static int $retryNumber = 0;
20+
private static ?DateTime $start = null;
21+
22+
protected function setUp(): void
23+
{
24+
self::$retryNumber = 0;
25+
self::$start = new DateTime;
26+
}
27+
28+
#[Retry(3)]
29+
public function testRetriesUntilMaxAttemptsThenSucceeds(): void
30+
{
31+
if (self::$retryNumber < 3) {
32+
self::$retryNumber++;
33+
34+
throw new Exception;
35+
}
36+
37+
$this->assertSame(3, self::$retryNumber);
38+
}
39+
40+
#[Retry(1)]
41+
public function testSingleRetryThenThrowsExpectedException(): void
42+
{
43+
if (self::$retryNumber < 1) {
44+
self::$retryNumber++;
45+
46+
throw new Exception;
47+
}
48+
49+
$this->expectException(Exception::class);
50+
$this->expectExceptionMessage('test exception two');
51+
$this->assertSame(1, self::$retryNumber);
52+
53+
throw new Exception('test exception two');
54+
}
55+
56+
#[Retry(2, 0, LogicException::class)]
57+
public function testRetryWithUnmatchedExceptionTypeFailsImmediately(): void
58+
{
59+
$this->expectException(Exception::class);
60+
$this->expectExceptionMessage('test exception');
61+
$this->assertSame(0, self::$retryNumber);
62+
self::$retryNumber++;
63+
64+
throw new Exception('test exception');
65+
}
66+
67+
#[Retry(2, 0, LogicException::class)]
68+
#[Retry(2)]
69+
public function testMultipleRetryAttributesFallBackToDefaultRetry(): void
70+
{
71+
if (self::$retryNumber < 2) {
72+
self::$retryNumber++;
73+
74+
throw new Exception;
75+
}
76+
77+
$this->assertSame(2, self::$retryNumber);
78+
}
79+
80+
#[Retry(5, 0, LogicException::class)]
81+
public function testRetriesUntilLogicExceptionStopsThrowing(): void
82+
{
83+
if (self::$retryNumber < 5) {
84+
self::$retryNumber++;
85+
86+
throw new LogicException;
87+
}
88+
89+
$this->assertSame(5, self::$retryNumber);
90+
}
91+
92+
#[Retry(1, 2)]
93+
public function testRetryDelaysExecutionBySpecifiedSeconds(): void
94+
{
95+
$end = new DateTime;
96+
97+
if (self::$retryNumber < 1) {
98+
self::$retryNumber++;
99+
100+
throw new Exception;
101+
}
102+
103+
$diffInSeconds = $end->getTimestamp() - self::$start->getTimestamp();
104+
105+
$this->assertSame(1, self::$retryNumber);
106+
$this->assertSame(2, $diffInSeconds);
107+
}
108+
}

0 commit comments

Comments
 (0)