Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend UnitTestHelper with IsIgnoredTestMethod #7333

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Extend UnitTestHelper with IsIgnoredTestMethod #7333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5272b3b
Move logic to UnitTestHelper.
Jun 2, 2023
48dd97b
Rename
pavel-mikula-sonarsource Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 5 additions & 27 deletions analyzers/src/SonarAnalyzer.CSharp/Rules/TestMethodShouldContainAssertion.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,11 @@ protected override void Initialize(SonarAnalysisContext context) =>
var methodDeclaration = MethodDeclarationFactory.Create(c.Node);
if (!methodDeclaration.Identifier.IsMissing
&& methodDeclaration.HasImplementation
&& c.SemanticModel.GetDeclaredSymbol(c.Node) is IMethodSymbol methodSymbol
&& IsTestMethod(methodSymbol, methodDeclaration.IsLocal)
&& !methodSymbol.HasExpectedExceptionAttribute()
&& !methodSymbol.HasAssertionInAttribute()
&& !IsTestIgnored(methodSymbol)
&& c.SemanticModel.GetDeclaredSymbol(c.Node) is IMethodSymbol method
&& method.IsTestMethod()
&& !method.HasExpectedExceptionAttribute()
&& !method.HasAssertionInAttribute()
&& !method.IsIgnoredTestMethod()
&& !ContainsAssertion(c.Node, c.SemanticModel, new HashSet<IMethodSymbol>(), 0))
{
c.ReportIssue(Diagnostic.Create(Rule, methodDeclaration.Identifier.GetLocation()));
Expand All @@ -76,13 +76,6 @@ protected override void Initialize(SonarAnalysisContext context) =>
SyntaxKind.MethodDeclaration,
SyntaxKindEx.LocalFunctionStatement);

// only xUnit allows local functions to be test methods.
private static bool IsTestMethod(IMethodSymbol symbol, bool isLocalFunction) =>
isLocalFunction ? IsXunitTestMethod(symbol) : symbol.IsTestMethod();

private static bool IsXunitTestMethod(IMethodSymbol methodSymbol) =>
methodSymbol.AnyAttributeDerivesFromAny(UnitTestHelper.KnownTestMethodAttributesOfxUnit);

private static bool ContainsAssertion(SyntaxNode methodDeclaration, SemanticModel previousSemanticModel, ISet<IMethodSymbol> visitedSymbols, int level)
{
var currentSemanticModel = methodDeclaration.EnsureCorrectSemanticModelOrDefault(previousSemanticModel);
Expand Down Expand Up @@ -125,21 +118,6 @@ private static bool ContainsAssertion(SyntaxNode methodDeclaration, SemanticMode
return false;
}

private static bool IsTestIgnored(IMethodSymbol method)
{
if (method.IsMsTestOrNUnitTestIgnored())
{
return true;
}

// Checking whether an Xunit test is ignore or not needs to be done at the syntax level i.e. language-specific
var factAttributeSyntax = method.FindXUnitTestAttribute()
?.ApplicationSyntaxReference.GetSyntax() as AttributeSyntax;

return factAttributeSyntax?.ArgumentList != null
&& factAttributeSyntax.ArgumentList.Arguments.Any(x => x.NameEquals.Name.Identifier.ValueText == "Skip");
}

private static bool IsAssertion(InvocationExpressionSyntax invocation) =>
invocation.Expression
.ToString()
Expand Down
220 changes: 114 additions & 106 deletions analyzers/src/SonarAnalyzer.Common/Helpers/UnitTestHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,111 +18,119 @@
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

namespace SonarAnalyzer.Helpers
namespace SonarAnalyzer.Helpers;

// Note: useful comparison of the differing syntax across unit test frameworks at https://xunit.net/docs/comparisons
internal static class UnitTestHelper
{
// Note: useful comparison of the differing syntax across unit test frameworks at https://xunit.net/docs/comparisons
internal static class UnitTestHelper
{
public static readonly ImmutableArray<KnownType> KnownTestMethodAttributesOfMSTest = ImmutableArray.Create(
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_TestMethodAttribute,
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_DataTestMethodAttribute);

public static readonly ImmutableArray<KnownType> KnownTestMethodAttributesOfNUnit = ImmutableArray.Create(
KnownType.NUnit_Framework_TestAttribute,
KnownType.NUnit_Framework_TestCaseAttribute,
KnownType.NUnit_Framework_TestCaseSourceAttribute,
KnownType.NUnit_Framework_TheoryAttribute,
KnownType.NUnit_Framework_ITestBuilderInterface);

public static readonly ImmutableArray<KnownType> KnownTestMethodAttributesOfxUnit = ImmutableArray.Create(
KnownType.Xunit_TheoryAttribute,
KnownType.LegacyXunit_TheoryAttribute,
// In order for the FindFirstTestMethodType to work, FactAttribute should go last as the Theory attribute derives from it.
KnownType.Xunit_FactAttribute);

public static readonly ImmutableArray<KnownType> KnownExpectedExceptionAttributes = ImmutableArray.Create(
// Note: XUnit doesn't have a exception attribute
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_ExpectedExceptionAttribute,
KnownType.NUnit_Framework_ExpectedExceptionAttribute);

public static readonly ImmutableArray<KnownType> KnownIgnoreAttributes = ImmutableArray.Create(
// Note: XUnit doesn't have a separate "Ignore" attribute. It has a "Skip" parameter
// on the test attribute
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_IgnoreAttribute,
KnownType.NUnit_Framework_IgnoreAttribute);

/// <summary>
/// List of partial names that are assumed to indicate an assertion method.
/// </summary>
public static readonly ImmutableArray<string> KnownAssertionMethodParts = ImmutableArray.Create(
"ASSERT",
"CHECK",
"EXPECT",
"MUST",
"SHOULD",
"VERIFY",
"VALIDATE");

private static readonly ImmutableArray<KnownType> KnownTestMethodAttributes = ImmutableArray.Create(
KnownTestMethodAttributesOfMSTest
.Concat(KnownTestMethodAttributesOfNUnit)
.Concat(KnownTestMethodAttributesOfxUnit)
.ToArray());

private static readonly ImmutableArray<KnownType> KnownTestClassAttributes = ImmutableArray.Create(
// xUnit does not have have attributes to identity test classes
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_TestClassAttribute,
KnownType.NUnit_Framework_TestFixtureAttribute);

private static readonly ImmutableArray<KnownType> NoExpectedResultTestMethodReturnTypes = ImmutableArray.Create(
KnownType.Void,
KnownType.System_Threading_Tasks_Task);

/// <summary>
/// Returns whether the class has an attribute that marks the class
/// as an MSTest or NUnit test class (xUnit doesn't have any such attributes).
/// </summary>
public static bool IsTestClass(this INamedTypeSymbol classSymbol) =>
classSymbol.AnyAttributeDerivesFromAny(KnownTestClassAttributes);

public static bool IsTestMethod(this IMethodSymbol method) =>
method.AnyAttributeDerivesFromOrImplementsAny(KnownTestMethodAttributes);

public static bool HasExpectedExceptionAttribute(this IMethodSymbol method) =>
method.GetAttributes().Any(a =>
a.AttributeClass.IsAny(KnownExpectedExceptionAttributes)
|| a.AttributeClass.DerivesFrom(KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_ExpectedExceptionBaseAttribute));

public static bool HasAssertionInAttribute(this IMethodSymbol method) =>
!NoExpectedResultTestMethodReturnTypes.Any(method.ReturnType.Is)
&& method.GetAttributes().Any(IsAnyTestCaseAttributeWithExpectedResult);

public static bool IsMsTestOrNUnitTestIgnored(this IMethodSymbol method) =>
method.GetAttributes().Any(a => a.AttributeClass.IsAny(KnownIgnoreAttributes));

public static AttributeData FindXUnitTestAttribute(this IMethodSymbol method) =>
method.GetAttributes().FirstOrDefault(a =>
a.AttributeClass.Is(KnownType.Xunit_FactAttribute)
|| a.AttributeClass.Is(KnownType.Xunit_TheoryAttribute)
|| a.AttributeClass.Is(KnownType.LegacyXunit_TheoryAttribute));

/// <summary>
/// Returns the <see cref="KnownType"/> that indicates the type of the test method or
/// null if the method is not decorated with a known type.
/// </summary>
/// <remarks>We assume that a test is only marked with a single test attribute e.g.
/// not both [Fact] and [Theory]. If there are multiple attributes only one will be
/// returned.</remarks>
public static KnownType FindFirstTestMethodType(this IMethodSymbol method) =>
KnownTestMethodAttributes.FirstOrDefault(known =>
method.GetAttributes().Any(att => att.AttributeClass.DerivesFrom(known)));

private static bool IsAnyTestCaseAttributeWithExpectedResult(AttributeData a) =>
IsTestAttributeWithExpectedResult(a)
|| a.AttributeClass.Is(KnownType.NUnit_Framework_TestCaseSourceAttribute);

private static bool IsTestAttributeWithExpectedResult(AttributeData a) =>
a.AttributeClass.IsAny(KnownType.NUnit_Framework_TestCaseAttribute, KnownType.NUnit_Framework_TestAttribute)
&& a.NamedArguments.Any(arg => arg.Key == "ExpectedResult");
}
public static readonly ImmutableArray<KnownType> KnownTestMethodAttributesOfMSTest = ImmutableArray.Create(
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_TestMethodAttribute,
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_DataTestMethodAttribute);

public static readonly ImmutableArray<KnownType> KnownTestMethodAttributesOfNUnit = ImmutableArray.Create(
KnownType.NUnit_Framework_TestAttribute,
KnownType.NUnit_Framework_TestCaseAttribute,
KnownType.NUnit_Framework_TestCaseSourceAttribute,
KnownType.NUnit_Framework_TheoryAttribute,
KnownType.NUnit_Framework_ITestBuilderInterface);

public static readonly ImmutableArray<KnownType> KnownTestMethodAttributesOfxUnit = ImmutableArray.Create(
KnownType.Xunit_TheoryAttribute,
KnownType.LegacyXunit_TheoryAttribute,
// In order for the FindFirstTestMethodType to work, FactAttribute should go last as the Theory attribute derives from it.
KnownType.Xunit_FactAttribute);

public static readonly ImmutableArray<KnownType> KnownExpectedExceptionAttributes = ImmutableArray.Create(
// Note: XUnit doesn't have a exception attribute
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_ExpectedExceptionAttribute,
KnownType.NUnit_Framework_ExpectedExceptionAttribute);

public static readonly ImmutableArray<KnownType> KnownIgnoreAttributes = ImmutableArray.Create(
// Note: XUnit doesn't have a separate "Ignore" attribute. It has a "Skip" parameter
// on the test attribute
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_IgnoreAttribute,
KnownType.NUnit_Framework_IgnoreAttribute);

/// <summary>
/// List of partial names that are assumed to indicate an assertion method.
/// </summary>
public static readonly ImmutableArray<string> KnownAssertionMethodParts = ImmutableArray.Create(
"ASSERT",
"CHECK",
"EXPECT",
"MUST",
"SHOULD",
"VERIFY",
"VALIDATE");

private static readonly ImmutableArray<KnownType> KnownTestMethodAttributes = ImmutableArray.Create(
KnownTestMethodAttributesOfMSTest
.Concat(KnownTestMethodAttributesOfNUnit)
.Concat(KnownTestMethodAttributesOfxUnit)
.ToArray());

private static readonly ImmutableArray<KnownType> KnownTestClassAttributes = ImmutableArray.Create(
// xUnit does not have have attributes to identity test classes
KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_TestClassAttribute,
KnownType.NUnit_Framework_TestFixtureAttribute);

private static readonly ImmutableArray<KnownType> NoExpectedResultTestMethodReturnTypes = ImmutableArray.Create(
KnownType.Void,
KnownType.System_Threading_Tasks_Task);

/// <summary>
/// Returns whether the class has an attribute that marks the class
/// as an MSTest or NUnit test class (xUnit doesn't have any such attributes).
/// </summary>
public static bool IsTestClass(this INamedTypeSymbol classSymbol) =>
classSymbol.AnyAttributeDerivesFromAny(KnownTestClassAttributes);

public static bool IsTestMethod(this IMethodSymbol method) =>
method.MethodKind.HasFlag(MethodKindEx.LocalFunction)
? method.IsXunitTestMethod()
: method.AnyAttributeDerivesFromOrImplementsAny(KnownTestMethodAttributes);

public static bool IsIgnoredTestMethod(this IMethodSymbol method) =>
method.HasIgnoredAttribute()
|| method.FindXUnitTestAttribute().NamedArguments.Any(arg => arg.Key == "Skip");

public static bool HasExpectedExceptionAttribute(this IMethodSymbol method) =>
method.GetAttributes().Any(a =>
a.AttributeClass.IsAny(KnownExpectedExceptionAttributes)
|| a.AttributeClass.DerivesFrom(KnownType.Microsoft_VisualStudio_TestTools_UnitTesting_ExpectedExceptionBaseAttribute));

public static bool HasAssertionInAttribute(this IMethodSymbol method) =>
!NoExpectedResultTestMethodReturnTypes.Any(method.ReturnType.Is)
&& method.GetAttributes().Any(IsAnyTestCaseAttributeWithExpectedResult);

public static AttributeData FindXUnitTestAttribute(this IMethodSymbol method) =>
method.GetAttributes().FirstOrDefault(a =>
a.AttributeClass.Is(KnownType.Xunit_FactAttribute)
|| a.AttributeClass.Is(KnownType.Xunit_TheoryAttribute)
|| a.AttributeClass.Is(KnownType.LegacyXunit_TheoryAttribute));

/// <summary>
/// Returns the <see cref="KnownType"/> that indicates the type of the test method or
/// null if the method is not decorated with a known type.
/// </summary>
/// <remarks>We assume that a test is only marked with a single test attribute e.g.
/// not both [Fact] and [Theory]. If there are multiple attributes only one will be
/// returned.</remarks>
public static KnownType FindFirstTestMethodType(this IMethodSymbol method) =>
KnownTestMethodAttributes.FirstOrDefault(known =>
method.GetAttributes().Any(att => att.AttributeClass.DerivesFrom(known)));

private static bool IsAnyTestCaseAttributeWithExpectedResult(AttributeData a) =>
IsTestAttributeWithExpectedResult(a)
|| a.AttributeClass.Is(KnownType.NUnit_Framework_TestCaseSourceAttribute);

private static bool IsTestAttributeWithExpectedResult(AttributeData a) =>
a.AttributeClass.IsAny(KnownType.NUnit_Framework_TestCaseAttribute, KnownType.NUnit_Framework_TestAttribute)
&& a.NamedArguments.Any(arg => arg.Key == "ExpectedResult");

private static bool IsXunitTestMethod(this IMethodSymbol methodSymbol) =>
methodSymbol.AnyAttributeDerivesFromAny(KnownTestMethodAttributesOfxUnit);

private static bool HasIgnoredAttribute(this IMethodSymbol method) =>
method.GetAttributes().Any(a => a.AttributeClass.IsAny(KnownIgnoreAttributes));
}