From 1324384169e042b5e3d8ce7105978ba6e91a71d1 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Thu, 13 Oct 2022 18:29:41 +0200 Subject: [PATCH] EncloseInNamespace for ConcurrentAnalysis --- .../Rules/AsyncVoidMethodTest.cs | 1 - .../Rules/ClassAndMethodNameTest.cs | 1 - .../Rules/CognitiveComplexityTest.cs | 1 - .../Rules/EmptyNamespaceTest.cs | 2 - ...onMethodShouldBeInSeparateNamespaceTest.cs | 1 - .../RequestsWithExcessiveLengthTest.cs | 1 - .../StringLiteralShouldNotBeDuplicatedTest.cs | 1 - .../LocksReleasedAllPathsTest.cs | 3 +- .../TestFramework/Tests/VerifierTest.cs | 530 ++++++++++++++++++ .../TestFramework/Verifier.cs | 280 +++++++++ 10 files changed, 811 insertions(+), 10 deletions(-) create mode 100644 analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Tests/VerifierTest.cs create mode 100644 analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Verifier.cs diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/AsyncVoidMethodTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/AsyncVoidMethodTest.cs index df259119fc8..94d4f94a118 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/AsyncVoidMethodTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/AsyncVoidMethodTest.cs @@ -60,7 +60,6 @@ public void AsyncVoidMethod_MsTestV2_CSharp11(string testFwkVersion) => // The first version of the framework is not compatible with Net 7 so we need to test only v2 with C#11 features .WithOptions(ParseOptionsHelper.FromCSharp11) .AddReferences(NuGetMetadataReference.MSTestTestFramework(testFwkVersion)) - .WithConcurrentAnalysis(false) .Verify(); #endif diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/ClassAndMethodNameTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/ClassAndMethodNameTest.cs index 9d8fe319a9e..968b2d8de35 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/ClassAndMethodNameTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/ClassAndMethodNameTest.cs @@ -32,7 +32,6 @@ public class ClassAndMethodNameTest public void ClassAndMethodName_CS() => builderCS.AddPaths("ClassAndMethodName.cs", "ClassAndMethodName.Partial.cs") .AddReferences(MetadataReferenceFacade.NetStandard21) - .WithConcurrentAnalysis(false) .WithOptions(ParseOptionsHelper.FromCSharp8) .Verify(); diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/CognitiveComplexityTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/CognitiveComplexityTest.cs index 4653a4cf8cf..1ed141bb72e 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/CognitiveComplexityTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/CognitiveComplexityTest.cs @@ -55,7 +55,6 @@ public void CognitiveComplexity_CS_CSharp9() => public void CognitiveComplexity_CS_CSharp10() => builderCS.AddPaths("CognitiveComplexity.CSharp10.cs") .WithOptions(ParseOptionsHelper.FromCSharp10) - .WithConcurrentAnalysis(false) .Verify(); [TestMethod] diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/EmptyNamespaceTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/EmptyNamespaceTest.cs index 97e873b1bc2..6be8e93c207 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/EmptyNamespaceTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/EmptyNamespaceTest.cs @@ -37,7 +37,6 @@ public void EmptyNamespace() => public void EmptyNamespace_CSharp10() => builder.AddPaths("EmptyNamespace.CSharp10.Empty.cs", "EmptyNamespace.CSharp10.NotEmpty.cs") .WithOptions(ParseOptionsHelper.FromCSharp10) - .WithConcurrentAnalysis(false) .Verify(); [TestMethod] @@ -45,7 +44,6 @@ public void EmptyNamespace_CSharp10_CodeFix() => builder.AddPaths("EmptyNamespace.CSharp10.Empty.cs") .WithCodeFix() .WithOptions(ParseOptionsHelper.FromCSharp10) - .WithAutogenerateConcurrentFiles(false) .WithCodeFixedPaths("EmptyNamespace.CSharp10.Fixed.cs") .VerifyCodeFix(); diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/ExtensionMethodShouldBeInSeparateNamespaceTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/ExtensionMethodShouldBeInSeparateNamespaceTest.cs index 44c5ed805ef..46a2cba8047 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/ExtensionMethodShouldBeInSeparateNamespaceTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/ExtensionMethodShouldBeInSeparateNamespaceTest.cs @@ -43,7 +43,6 @@ public void ExtensionMethodShouldBeInSeparateNamespace_CSharp9() => public void ExtensionMethodShouldBeInSeparateNamespace_CSharp10() => builder .AddPaths("ExtensionMethodShouldBeInSeparateNamespace.CSharp10.cs") - .WithConcurrentAnalysis(false) .WithOptions(ParseOptionsHelper.FromCSharp10) .Verify(); diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/Hotspots/RequestsWithExcessiveLengthTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/Hotspots/RequestsWithExcessiveLengthTest.cs index 50c5f0aa15e..6cda1dea0fe 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/Hotspots/RequestsWithExcessiveLengthTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/Hotspots/RequestsWithExcessiveLengthTest.cs @@ -65,7 +65,6 @@ public void RequestsWithExcessiveLength_Csharp10() => public void RequestsWithExcessiveLength_Csharp11() => builderCS .AddPaths(@"RequestsWithExcessiveLength.CSharp11.cs") - .WithConcurrentAnalysis(false) .WithOptions(ParseOptionsHelper.FromCSharp11).Verify(); #endif diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/StringLiteralShouldNotBeDuplicatedTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/StringLiteralShouldNotBeDuplicatedTest.cs index ee018323b2b..261fb5dd2ea 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/StringLiteralShouldNotBeDuplicatedTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/StringLiteralShouldNotBeDuplicatedTest.cs @@ -64,7 +64,6 @@ public void StringLiteralShouldNotBeDuplicated_CSharp11() => public void StringLiteralShouldNotBeDuplicated_Attributes_CS() => new VerifierBuilder().AddAnalyzer(() => new CS.StringLiteralShouldNotBeDuplicated { Threshold = 2 }) .AddPaths("StringLiteralShouldNotBeDuplicated_Attributes.cs") - .WithConcurrentAnalysis(false) .Verify(); [TestMethod] diff --git a/analyzers/tests/SonarAnalyzer.Test/Rules/SymbolicExecution/LocksReleasedAllPathsTest.cs b/analyzers/tests/SonarAnalyzer.Test/Rules/SymbolicExecution/LocksReleasedAllPathsTest.cs index 2f1b78af6ab..bd2f93c7a7b 100644 --- a/analyzers/tests/SonarAnalyzer.Test/Rules/SymbolicExecution/LocksReleasedAllPathsTest.cs +++ b/analyzers/tests/SonarAnalyzer.Test/Rules/SymbolicExecution/LocksReleasedAllPathsTest.cs @@ -87,7 +87,6 @@ private static VerifierBuilder CreateVerifier(Func createCon .AddAnalyzer(createConfiguredAnalyzer) .WithOnlyDiagnostics(onlyDiagnostics) .AddReferences(MetadataReferenceFacade.SystemThreading) - .WithBasePath(@"SymbolicExecution\Roslyn") - .WithConcurrentAnalysis(false); + .WithBasePath(@"SymbolicExecution\Roslyn"); } } diff --git a/analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Tests/VerifierTest.cs b/analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Tests/VerifierTest.cs new file mode 100644 index 00000000000..8e6e4b3db08 --- /dev/null +++ b/analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Tests/VerifierTest.cs @@ -0,0 +1,530 @@ +/* + * SonarAnalyzer for .NET + * Copyright (C) 2015-2022 SonarSource SA + * mailto: contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +using System.IO; +using SonarAnalyzer.Protobuf; +using SonarAnalyzer.Rules.CSharp; +using SonarAnalyzer.SymbolicExecution.Sonar.Analyzers; + +namespace SonarAnalyzer.UnitTest.TestFramework.Tests +{ + [TestClass] + public class VerifierTest + { + private static readonly VerifierBuilder DummyCS = new VerifierBuilder(); + private static readonly VerifierBuilder DummyVB = new VerifierBuilder(); + private static readonly VerifierBuilder DummyCodeFixCS = new VerifierBuilder() + .AddPaths("Path.cs") + .WithCodeFix() + .WithCodeFixedPaths("Expected.cs"); + + public TestContext TestContext { get; set; } + + [TestMethod] + public void Constructor_Null_Throws() => + ((Func)(() => new(null))).Should().Throw().And.ParamName.Should().Be("builder"); + + [TestMethod] + public void Constructor_NoAnalyzers_Throws() => + new VerifierBuilder() + .Invoking(x => x.Build()).Should().Throw() + .WithMessage("Analyzers cannot be empty. Use VerifierBuilder instead or add at least one analyzer using builder.AddAnalyzer()."); + + [TestMethod] + public void Constructor_NullAnalyzers_Throws() => + new VerifierBuilder().AddAnalyzer(() => null) + .Invoking(x => x.Build()).Should().Throw().WithMessage("Analyzer instance cannot be null."); + + [TestMethod] + public void Constructor_NoPaths_Throws() => + DummyCS.Invoking(x => x.Build()).Should().Throw().WithMessage("Paths cannot be empty. Add at least one file using builder.AddPaths() or AddSnippet()."); + + [TestMethod] + public void Constructor_MixedLanguageAnalyzers_Throws() => + DummyCS.AddAnalyzer(() => new SonarAnalyzer.Rules.VisualBasic.OptionStrictOn()) + .Invoking(x => x.Build()).Should().Throw().WithMessage("All Analyzers must declare the same language in their DiagnosticAnalyzerAttribute."); + + [TestMethod] + public void Constructor_MixedLanguagePaths_Throws() => + DummyCS.AddPaths("File.txt") + .Invoking(x => x.Build()).Should().Throw().WithMessage("Path 'File.txt' doesn't match C# file extension '.cs'."); + + [TestMethod] + public void Constructor_CodeFix_MissingCodeFixedPath_Throws() => + DummyCodeFixCS.WithCodeFixedPaths(null) + .Invoking(x => x.Build()).Should().Throw().WithMessage("CodeFixedPath was not set."); + + [TestMethod] + public void Constructor_CodeFix_WrongCodeFixedPath_Throws() => + DummyCodeFixCS.WithCodeFixedPaths("File.vb") + .Invoking(x => x.Build()).Should().Throw().WithMessage("Path 'File.vb' doesn't match C# file extension '.cs'."); + + [TestMethod] + public void Constructor_CodeFix_WrongCodeFixedPathBatch_Throws() => + DummyCodeFixCS.WithCodeFixedPaths("File.cs", "Batch.vb") + .Invoking(x => x.Build()).Should().Throw().WithMessage("Path 'Batch.vb' doesn't match C# file extension '.cs'."); + + [TestMethod] + public void Constructor_CodeFix_MultipleAnalyzers_Throws() => + DummyCodeFixCS.AddAnalyzer(() => new DummyAnalyzerCS()) + .Invoking(x => x.Build()).Should().Throw().WithMessage("When CodeFix is set, Analyzers must contain only 1 analyzer, but 2 were found."); + + [TestMethod] + public void Constructor_CodeFix_MultiplePaths_Throws() => + DummyCodeFixCS.AddPaths("Second.cs", "Third.cs") + .Invoking(x => x.Build()).Should().Throw().WithMessage("Paths must contain only 1 file, but 3 were found."); + + [TestMethod] + public void Constructor_CodeFix_WithSnippets_Throws() => + DummyCodeFixCS.AddSnippet("Wrong") + .Invoking(x => x.Build()).Should().Throw().WithMessage("Snippets must be empty when CodeFix is set."); + + [TestMethod] + public void Constructor_CodeFix_WrongLanguage_Throws() => + DummyCodeFixCS.WithCodeFix() + .Invoking(x => x.Build()).Should().Throw().WithMessage("DummyAnalyzerCS language C# does not match DummyCodeFixVB language."); + + [TestMethod] + public void Constructor_CodeFix_FixableDiagnosticsNotSupported_Throws() => + DummyCodeFixCS.WithCodeFix() + .Invoking(x => x.Build()).Should().Throw().WithMessage("DummyAnalyzerCS does not support diagnostics fixable by the EmptyMethodCodeFix."); + + [TestMethod] + public void Constructor_CodeFix_MissingAttribute_Throws() => + DummyCodeFixCS.WithCodeFix() + .Invoking(x => x.Build()).Should().Throw().WithMessage("DummyCodeFixNoAttribute does not have ExportCodeFixProviderAttribute."); + + [TestMethod] + public void Constructor_ProtobufPath_MultipleAnalyzers_Throws() => + DummyCS.AddSnippet("//Empty").WithProtobufPath("Proto.pb").AddAnalyzer(() => new DummyAnalyzerCS()) + .Invoking(x => x.Build()).Should().Throw().WithMessage("When ProtobufPath is set, Analyzers must contain only 1 analyzer, but 2 were found."); + + [TestMethod] + public void Constructor_ProtobufPath_WrongAnalyzerType_Throws() => + DummyCS.AddSnippet("//Empty").WithProtobufPath("Proto.pb") + .Invoking(x => x.Build()).Should().Throw().WithMessage("DummyAnalyzerCS does not inherit from UtilityAnalyzerBase."); + + [TestMethod] + public void Verify_ThrowsWithCodeFixSet() + { + var originalPath = WriteFile("File.cs", null); + var fixedPath = WriteFile("File.Fixed.cs", null); + DummyCS.AddPaths(originalPath).WithCodeFix().WithCodeFixedPaths(fixedPath).Invoking(x => x.Verify()) + .Should().Throw().WithMessage("Cannot use Verify with CodeFix set."); + } + + [TestMethod] + public void Verify_RaiseExpectedIssues_CS() => + WithSnippetCS( +@"public class Sample +{ + private int a = 42; // Noncompliant {{Message for SDummy}} + private int b = 42; // Noncompliant + private bool c = true; +}").Invoking(x => x.Verify()).Should().NotThrow(); + + [TestMethod] + public void Verify_RaiseExpectedIssues_VB() => + WithSnippetVB( +@"Public Class Sample + Private A As Integer = 42 ' Noncompliant {{Message for SDummy}} + Private B As Integer = 42 ' Noncompliant + Private C As Boolean = True +End Class").Invoking(x => x.Verify()).Should().NotThrow(); + + [TestMethod] + public void Verify_RaiseUnexpectedIssues_CS() => + WithSnippetCS( +@"public class Sample +{ + private int a = 42; // FP + private int b = 42; // FP + private bool c = true; +}").Invoking(x => x.Verify()).Should().Throw().WithMessage("CSharp7: Unexpected primary issue on line 3, span (2,20)-(2,22) with message 'Message for SDummy'*"); + + [TestMethod] + public void Verify_RaiseUnexpectedIssues_VB() => + WithSnippetVB( +@"Public Class Sample + Private A As Integer = 42 ' FP + Private B As Integer = 42 ' FP + Private C As Boolean = True +End Class").Invoking(x => x.Verify()).Should().Throw().WithMessage("VisualBasic12: Unexpected primary issue on line 2, span (1,27)-(1,29) with message 'Message for SDummy'*"); + + [TestMethod] + public void Verify_MissingExpectedIssues() => + WithSnippetCS( +@"public class Sample +{ + private bool a = true; // Noncompliant - FN + private bool b = true; // Noncompliant - FN + private bool c = true; +}").Invoking(x => x.Verify()).Should().Throw().WithMessage( +@"CSharp7: Issue(s) expected but not raised in file(s): +File: File.cs +Line: 3, Type: primary, Id: '' +Line: 4, Type: primary, Id: '' + +File: File.Concurrent.cs +Line: 3, Type: primary, Id: '' +Line: 4, Type: primary, Id: '' +"); + + [TestMethod] + public void Verify_TwoAnalyzers() => + WithSnippetCS( +@"public class Sample +{ + private int a = 42; // Noncompliant {{Message for SDummy}} + // Noncompliant@-1 + private int b = 42; // Noncompliant + // Noncompliant@-1 + private bool c = true; +}") + .AddAnalyzer(() => new DummyAnalyzerCS()) // Duplicate + .Invoking(x => x.Verify()).Should().NotThrow(); + + [TestMethod] + public void Verify_TwoPaths() => + WithSnippetCS( +@"public class First +{ + private bool a = true; // Noncompliant - FN in File.cs +}") + .AddPaths(WriteFile("Second.cs", +@"public class Second +{ + private bool a = true; // Noncompliant - FN in Second.cs +}")) + .WithConcurrentAnalysis(false) + .Invoking(x => x.Verify()).Should().Throw().WithMessage( +@"CSharp7: Issue(s) expected but not raised in file(s): +File: File.cs +Line: 3, Type: primary, Id: '' + +File: Second.cs +Line: 3, Type: primary, Id: '' +"); + + [TestMethod] + public void Verify_AutogenerateConcurrentFiles() + { + var builder = WithSnippetCS( +@"namespace N1 { + // Noncompliant - FN +}"); + // Concurrent analysis by-default automatically generates concurrent files - File.Concurrent.cs + builder.Invoking(x => x.Verify()).Should().Throw().WithMessage( + @"CSharp7: Issue(s) expected but not raised in file(s): +File: File.cs +Line: 2, Type: primary, Id: '' + +File: File.Concurrent.cs +Line: 2, Type: primary, Id: '' +"); + // When AutogenerateConcurrentFiles is turned off, only the provided snippet is analyzed + builder.WithAutogenerateConcurrentFiles(false).Invoking(x => x.Verify()).Should().Throw().WithMessage( + @"CSharp7: Issue(s) expected but not raised in file(s): +File: File.cs +Line: 2, Type: primary, Id: '' +"); + } + + [TestMethod] + public void Verify_TestProject() + { + var builder = new VerifierBuilder() // Rule with scope Main + .AddSnippet("public class Sample { public void Main() { System.Console.WriteLine(); } }"); + builder.Invoking(x => x.Verify()).Should().Throw(); + builder.AddTestReference().Invoking(x => x.Verify()).Should().NotThrow("Project references should be recognized as Test code."); + } + + [TestMethod] + public void Verify_ParseOptions() + { + var builder = WithSnippetCS( +@"public class Sample +{ + private System.Exception ex = new(); // C# 9 target-typed new +}"); + builder.WithOptions(ParseOptionsHelper.FromCSharp9).Invoking(x => x.Verify()).Should().NotThrow(); + builder.WithOptions(ParseOptionsHelper.BeforeCSharp9).Invoking(x => x.Verify()).Should().Throw() + .WithMessage("CSharp5: Unexpected build error [CS8026]: Feature 'target-typed object creation' is not available in C# 5. Please use language version 9.0 or greater. on line 3"); + } + + [TestMethod] + public void Verify_BasePath() + { + DummyCS.AddPaths("Nonexistent.cs").Invoking(x => x.Verify()).Should().Throw("This file should not exist in TestCases directory."); + DummyCS.AddPaths("ArrayCovariance.cs").Invoking(x => x.Verify()).Should().Throw("File should be found in TestCases directory."); + DummyCS.WithBasePath("TestFramework").AddPaths("Verifier.BasePath.cs").Invoking(x => x.Verify()).Should().NotThrow(); + } + + [TestMethod] + public void Verify_ErrorBehavior() + { + var builder = WithSnippetCS("undefined"); + builder.Invoking(x => x.Verify()).Should().Throw() + .WithMessage("CSharp7: Unexpected build error [CS0116]: A namespace cannot directly contain members such as fields, methods or statements on line 1"); + builder.WithErrorBehavior(CompilationErrorBehavior.FailTest).Invoking(x => x.Verify()).Should().Throw() + .WithMessage("CSharp7: Unexpected build error [CS0116]: A namespace cannot directly contain members such as fields, methods or statements on line 1"); + builder.WithErrorBehavior(CompilationErrorBehavior.Ignore).Invoking(x => x.Verify()).Should().NotThrow(); + } + + [TestMethod] + public void Verify_OnlyDiagnostics() + { + var builder = new VerifierBuilder().AddPaths(WriteFile("File.cs", +@"public class Sample +{ + public void Method() + { + var t = true; + if (t) // S2583 + t = true; + else + t = true; + if (t) // S2589 + t = true; + } +}")); + builder.Invoking(x => x.Verify()).Should().Throw().WithMessage( +@"CSharp7: Unexpected primary issue on line 6, span (5,12)-(5,13) with message 'Change this condition so that it does not always evaluate to 'true'; some subsequent code is never executed.'.*"); + builder.WithOnlyDiagnostics(ConditionEvaluatesToConstant.S2589).Invoking(x => x.Verify()).Should().Throw().WithMessage( +@"CSharp7: Unexpected primary issue on line 10, span (9,12)-(9,13) with message 'Change this condition so that it does not always evaluate to 'true'.'*"); + builder.WithOnlyDiagnostics(NullPointerDereference.S2259).Invoking(x => x.Verify()).Should().NotThrow(); + } + + [TestMethod] + public void Verify_NonConcurrentAnalysis() + { + var builder = WithSnippetCS("var topLevelStatement = true;").WithOptions(ParseOptionsHelper.FromCSharp9).WithOutputKind(OutputKind.ConsoleApplication); + builder.Invoking(x => x.Verify()).Should().Throw("Default Verifier behavior duplicates the source file.") + .WithMessage("CSharp9: Unexpected build error [CS0825]: The contextual keyword 'var' may only appear within a local variable declaration or in script code on line 1"); + builder.WithConcurrentAnalysis(false).Invoking(x => x.Verify()).Should().NotThrow(); + } + + [TestMethod] + public void Verify_OutputKind() + { + var builder = WithSnippetCS("var topLevelStatement = true;").WithOptions(ParseOptionsHelper.FromCSharp9); + builder.WithTopLevelStatements().Invoking(x => x.Verify()).Should().NotThrow(); + builder.WithOutputKind(OutputKind.ConsoleApplication).WithConcurrentAnalysis(false).Invoking(x => x.Verify()).Should().NotThrow(); + builder.Invoking(x => x.Verify()).Should().Throw() + .WithMessage("CSharp9: Unexpected build error [CS8805]: Program using top-level statements must be an executable. on line 1"); + } + + [TestMethod] + public void Verify_Snippets() => + DummyCS.AddSnippet("public class First { } // Noncompliant [first] - not raised") + .AddSnippet("public class Second { } // Noncompliant [second] - not raised") + .Invoking(x => x.Verify()).Should().Throw().WithMessage( +@"CSharp7: Issue(s) expected but not raised in file(s): +File: snippet1.cs +Line: 1, Type: primary, Id: 'first' + +File: snippet2.cs +Line: 1, Type: primary, Id: 'second' +"); + + [TestMethod] + public void VerifyCodeFix_FixExpected_CS() + { + var originalPath = WriteFile("File.cs", +@"public class Sample +{ + private int a = 0; // Noncompliant + private int b = 0; // Noncompliant + private bool c = true; +}"); + var fixedPath = WriteFile("File.Fixed.cs", +@"public class Sample +{ + private int a = default; // Fixed + private int b = default; // Fixed + private bool c = true; +}"); + DummyCS.AddPaths(originalPath).WithCodeFix().WithCodeFixedPaths(fixedPath).Invoking(x => x.VerifyCodeFix()).Should().NotThrow(); + } + + [TestMethod] + public void VerifyCodeFix_FixExpected_VB() + { + var originalPath = WriteFile("File.vb", +@"Public Class Sample + Private A As Integer = 42 ' Noncompliant + Private B As Integer = 42 ' Noncompliant + Private C As Boolean = True +End Class"); + var fixedPath = WriteFile("File.Fixed.vb", +@"Public Class Sample + Private A As Integer = Nothing ' Fixed + Private B As Integer = Nothing ' Fixed + Private C As Boolean = True +End Class"); + DummyVB.AddPaths(originalPath).WithCodeFix().WithCodeFixedPaths(fixedPath).Invoking(x => x.VerifyCodeFix()).Should().NotThrow(); + } + + [TestMethod] + public void VerifyCodeFix_NotFixed_CS() + { + var originalPath = WriteFile("File.cs", +@"public class Sample +{ + private int a = 0; // Noncompliant + private int b = 0; // Noncompliant + private bool c = true; +}"); + DummyCS.AddPaths(originalPath).WithCodeFix().WithCodeFixedPaths(originalPath).Invoking(x => x.VerifyCodeFix()).Should().Throw().WithMessage( +@"Expected * to be* +""public class Sample +{ + private int a = 0; // Noncompliant + private int b = 0; // Noncompliant + private bool c = true; +}"" with a length of 136 because VerifyWhileDocumentChanges updates the document until all issues are fixed, even if the fix itself creates a new issue again. Language: CSharp7, but* +""public class Sample +{ + private int a = default; // Fixed + private int b = default; // Fixed + private bool c = true; +}"" has a length of 134, differs near ""def"" (index 42)."); + } + + [TestMethod] + public void VerifyCodeFix_NotFixed_VB() + { + var originalPath = WriteFile("File.vb", +@"Public Class Sample + Private A As Integer = 42 ' Noncompliant + Private B As Integer = 42 ' Noncompliant + Private C As Boolean = True +End Class"); + DummyVB.AddPaths(originalPath).WithCodeFix().WithCodeFixedPaths(originalPath).Invoking(x => x.VerifyCodeFix()).Should().Throw().WithMessage( +@"Expected * to be* +""Public Class Sample + Private A As Integer = 42 ' Noncompliant + Private B As Integer = 42 ' Noncompliant + Private C As Boolean = True +End Class"" with a length of 155 because VerifyWhileDocumentChanges updates the document until all issues are fixed, even if the fix itself creates a new issue again. Language: VisualBasic12, but* +""Public Class Sample + Private A As Integer = Nothing ' Fixed + Private B As Integer = Nothing ' Fixed + Private C As Boolean = True +End Class"" has a length of 151, differs near ""Not"" (index 47)."); + } + + [TestMethod] + public void VerifyNoIssueReported_NoIssues_Succeeds() => + WithSnippetCS("// Noncompliant - this comment is ignored").Invoking(x => x.VerifyNoIssueReported()).Should().NotThrow(); + + [TestMethod] + public void VerifyNoIssueReported_WithIssues_Throws() => + WithSnippetCS( +@"public class Sample +{ + private int a = 42; // This will raise an issue +}").Invoking(x => x.VerifyNoIssueReported()).Should().Throw(); + + [TestMethod] + public void Verify_ConcurrentAnalysis_FileEndingWithComment_CS() => + WithSnippetCS("// Nothing to see here, file ends with a comment").Invoking(x => x.Verify()).Should().NotThrow(); + + [TestMethod] + public void Verify_ConcurrentAnalysis_FileEndingWithComment_VB() => + WithSnippetVB("' Nothing to see here, file ends with a comment").Invoking(x => x.Verify()).Should().NotThrow(); + + [TestMethod] + public void VerifyUtilityAnalyzerProducesEmptyProtobuf_EmptyFile() + { + var protobufPath = TestHelper.TestPath(TestContext, "Empty.pb"); + new VerifierBuilder().AddAnalyzer(() => new DummyUtilityAnalyzerCS(protobufPath, null)).AddSnippet("// Nothing to see here").WithProtobufPath(protobufPath) + .Invoking(x => x.VerifyUtilityAnalyzerProducesEmptyProtobuf()) + .Should().NotThrow(); + } + + [TestMethod] + public void VerifyUtilityAnalyzerProducesEmptyProtobuf_WithContent() + { + var protobufPath = TestHelper.TestPath(TestContext, "Empty.pb"); + var message = new LogInfo { Text = "Lorem Ipsum" }; + new VerifierBuilder().AddAnalyzer(() => new DummyUtilityAnalyzerCS(protobufPath, message)).AddSnippet("// Nothing to see here").WithProtobufPath(protobufPath) + .Invoking(x => x.VerifyUtilityAnalyzerProducesEmptyProtobuf()) + .Should().Throw().WithMessage("Expected value to be 0L because protobuf file should be empty, but found *"); + } + + [TestMethod] + public void VerifyUtilityAnalyzer_CorrectProtobuf_CS() + { + var protobufPath = TestHelper.TestPath(TestContext, "Log.pb"); + var message = new LogInfo { Text = "Lorem Ipsum" }; + var wasInvoked = false; + new VerifierBuilder().AddAnalyzer(() => new DummyUtilityAnalyzerCS(protobufPath, message)).AddSnippet("// Nothing to see here").WithProtobufPath(protobufPath) + .VerifyUtilityAnalyzer(x => + { + x.Should().ContainSingle().Which.Text.Should().Be("Lorem Ipsum"); + wasInvoked = true; + }); + wasInvoked.Should().BeTrue(); + } + + [TestMethod] + public void VerifyUtilityAnalyzer_CorrectProtobuf_VB() + { + var protobufPath = TestHelper.TestPath(TestContext, "Log.pb"); + var message = new LogInfo { Text = "Lorem Ipsum" }; + var wasInvoked = false; + new VerifierBuilder().AddAnalyzer(() => new DummyUtilityAnalyzerVB(protobufPath, message)).AddSnippet("' Nothing to see here").WithProtobufPath(protobufPath) + .VerifyUtilityAnalyzer(x => + { + x.Should().ContainSingle().Which.Text.Should().Be("Lorem Ipsum"); + wasInvoked = true; + }); + wasInvoked.Should().BeTrue(); + } + + [TestMethod] + public void VerifyUtilityAnalyzer_VerifyProtobuf_PropagateFailedAssertion_CS() + { + var protobufPath = TestHelper.TestPath(TestContext, "Empty.pb"); + new VerifierBuilder().AddAnalyzer(() => new DummyUtilityAnalyzerCS(protobufPath, null)).AddSnippet("// Nothing to see here").WithProtobufPath(protobufPath) + .Invoking(x => x.VerifyUtilityAnalyzer(x => throw new AssertFailedException("Some failed assertion about Protobuf"))) + .Should().Throw().WithMessage("Some failed assertion about Protobuf"); + } + + [TestMethod] + public void VerifyUtilityAnalyzer_VerifyProtobuf_PropagateFailedAssertion_VB() + { + var protobufPath = TestHelper.TestPath(TestContext, "Empty.pb"); + new VerifierBuilder().AddAnalyzer(() => new DummyUtilityAnalyzerVB(protobufPath, null)).AddSnippet("' Nothing to see here").WithProtobufPath(protobufPath) + .Invoking(x => x.VerifyUtilityAnalyzer(x => throw new AssertFailedException("Some failed assertion about Protobuf"))) + .Should().Throw().WithMessage("Some failed assertion about Protobuf"); + } + + private VerifierBuilder WithSnippetCS(string code) => + DummyCS.AddPaths(WriteFile("File.cs", code)); + + private VerifierBuilder WithSnippetVB(string code) => + DummyVB.AddPaths(WriteFile("File.vb", code)); + + private string WriteFile(string name, string content) => + TestHelper.WriteFile(TestContext, name, content); + } +} diff --git a/analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Verifier.cs b/analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Verifier.cs new file mode 100644 index 00000000000..b5826463e0c --- /dev/null +++ b/analyzers/tests/SonarAnalyzer.UnitTest/TestFramework/Verifier.cs @@ -0,0 +1,280 @@ +/* + * SonarAnalyzer for .NET + * Copyright (C) 2015-2022 SonarSource SA + * mailto: contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +using System.IO; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using Google.Protobuf; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using SonarAnalyzer.Common; +using SonarAnalyzer.Rules; +using SonarAnalyzer.UnitTest.Helpers; + +namespace SonarAnalyzer.UnitTest.TestFramework +{ + internal class Verifier + { + private const string TestCases = "TestCases"; + + private static readonly Regex ImportsRegexVB = new(@"^\s*Imports\s+.+$", RegexOptions.Multiline | RegexOptions.RightToLeft); + private readonly VerifierBuilder builder; + private readonly DiagnosticAnalyzer[] analyzers; + private readonly SonarCodeFix codeFix; + private readonly AnalyzerLanguage language; + private readonly string[] onlyDiagnosticIds; + + public Verifier(VerifierBuilder builder) + { + this.builder = builder ?? throw new ArgumentNullException(nameof(builder)); + onlyDiagnosticIds = builder.OnlyDiagnostics.Select(x => x.Id).ToArray(); + analyzers = builder.Analyzers.Select(x => x()).ToArray(); + if (!analyzers.Any()) + { + throw new ArgumentException($"{nameof(builder.Analyzers)} cannot be empty. Use {nameof(VerifierBuilder)} instead or add at least one analyzer using {nameof(builder)}.{nameof(builder.AddAnalyzer)}()."); + } + if (analyzers.Any(x => x == null)) + { + throw new ArgumentException("Analyzer instance cannot be null."); + } + var allLanguages = analyzers.SelectMany(x => x.GetType().GetCustomAttributes()).SelectMany(x => x.Languages).Distinct().ToArray(); + if (allLanguages.Length > 1) + { + throw new ArgumentException($"All {nameof(builder.Analyzers)} must declare the same language in their DiagnosticAnalyzerAttribute."); + } + language = AnalyzerLanguage.FromName(allLanguages.Single()); + if (!builder.Paths.Any() && !builder.Snippets.Any()) + { + throw new ArgumentException($"{nameof(builder.Paths)} cannot be empty. Add at least one file using {nameof(builder)}.{nameof(builder.AddPaths)}() or {nameof(builder.AddSnippet)}()."); + } + foreach (var path in builder.Paths) + { + ValidateExtension(path); + } + if (builder.ProtobufPath is not null) + { + ValidateSingleAnalyzer(nameof(builder.ProtobufPath)); + if (analyzers.Single() is not UtilityAnalyzerBase) + { + throw new ArgumentException($"{analyzers.Single().GetType().Name} does not inherit from {nameof(UtilityAnalyzerBase)}."); + } + } + if (builder.CodeFix is not null) + { + codeFix = builder.CodeFix(); + ValidateCodeFix(); + } + } + + public void Verify() // This should never has any arguments + { + if (codeFix != null) + { + throw new InvalidOperationException($"Cannot use {nameof(Verify)} with {nameof(builder.CodeFix)} set."); + } + foreach (var compilation in Compile(builder.ConcurrentAnalysis)) + { + DiagnosticVerifier.Verify(compilation, analyzers, builder.ErrorBehavior, builder.SonarProjectConfigPath, onlyDiagnosticIds); + } + } + + public void VerifyNoIssueReported() // This should never has any arguments + { + foreach (var compilation in Compile(builder.ConcurrentAnalysis)) + { + foreach (var analyzer in analyzers) + { + DiagnosticVerifier.VerifyNoIssueReported(compilation, analyzer, builder.ErrorBehavior, builder.SonarProjectConfigPath, onlyDiagnosticIds); + } + } + } + + public void VerifyCodeFix() // This should never has any arguments + { + _ = codeFix ?? throw new InvalidOperationException($"{nameof(builder.CodeFix)} was not set."); + var document = CreateProject(false).FindDocument(Path.GetFileName(builder.Paths.Single())); + var codeFixVerifier = new CodeFixVerifier(analyzers.Single(), codeFix, document, builder.CodeFixTitle); + var fixAllProvider = codeFix.GetFixAllProvider(); + foreach (var parseOptions in builder.ParseOptions.OrDefault(language.LanguageName)) + { + codeFixVerifier.VerifyWhileDocumentChanges(parseOptions, TestCasePath(builder.CodeFixedPath)); + if (fixAllProvider is not null) + { + codeFixVerifier.VerifyFixAllProvider(fixAllProvider, parseOptions, TestCasePath(builder.CodeFixedPathBatch ?? builder.CodeFixedPath)); + } + } + } + + public void VerifyUtilityAnalyzerProducesEmptyProtobuf() // This should never has any arguments + { + foreach (var compilation in Compile(false)) + { + DiagnosticVerifier.Verify(compilation, analyzers.Single(), CompilationErrorBehavior.Default); + new FileInfo(builder.ProtobufPath).Length.Should().Be(0, "protobuf file should be empty"); + } + } + + public void VerifyUtilityAnalyzer(Action> verifyProtobuf) + where TMessage : IMessage, new() + { + foreach (var compilation in Compile(false)) + { + DiagnosticVerifier.Verify(compilation, analyzers.Single(), builder.ErrorBehavior, builder.SonarProjectConfigPath); + verifyProtobuf(ReadProtobuf().ToList()); + } + + IEnumerable ReadProtobuf() + { + using var input = File.OpenRead(builder.ProtobufPath); + var parser = new MessageParser(() => new TMessage()); + while (input.Position < input.Length) + { + yield return parser.ParseDelimitedFrom(input); + } + } + } + + public IEnumerable Compile(bool concurrentAnalysis) => + CreateProject(concurrentAnalysis).Solution.Compile(builder.ParseOptions.ToArray()); + + private ProjectBuilder CreateProject(bool concurrentAnalysis) + { + using var scope = new EnvironmentVariableScope { EnableConcurrentAnalysis = concurrentAnalysis }; + var paths = builder.Paths.Select(TestCasePath).ToArray(); + return SolutionBuilder.Create() + .AddProject(language, true, builder.OutputKind) + .AddDocuments(paths) + .AddDocuments(concurrentAnalysis && builder.AutogenerateConcurrentFiles ? CreateConcurrencyTest(paths) : Enumerable.Empty()) + .AddSnippets(builder.Snippets.ToArray()) + .AddReferences(builder.References); + } + + private IEnumerable CreateConcurrencyTest(IEnumerable paths) + { + foreach (var path in paths) + { + var newPath = Path.ChangeExtension(path, ".Concurrent" + language.FileExtension); + var content = File.ReadAllText(path, Encoding.UTF8); + File.WriteAllText(newPath, InsertConcurrentNamespace(content)); + yield return newPath; + } + } + + private string InsertConcurrentNamespace(string content) + { + return language.LanguageName switch + { + LanguageNames.CSharp => EncloseInNamespace(content), + LanguageNames.VisualBasic => content.Insert(ImportsIndexVB(), "Namespace AppendedNamespaceForConcurrencyTest : ") + Environment.NewLine + " : End Namespace", + _ => throw new UnexpectedLanguageException(language) + }; + + int ImportsIndexVB() => + ImportsRegexVB.Match(content) is { Success: true } match ? match.Index + match.Length + 1 : 0; + } + + private static string EncloseInNamespace(string content) + { + var tree = CSharpSyntaxTree.ParseText(content); + if (tree.TryGetRoot(out var root) && root is CompilationUnitSyntax { Members: { } members } compilationUnit) + { + if (members.OfType().FirstOrDefault() is { } fileScoped) + { + root = root.ReplaceNode(fileScoped, fileScoped.WithName(SyntaxFactory.ParseName($"ConcurrencyTest.{CSharpSyntaxHelper.GetName(fileScoped.Name)}"))); + } + else + { + var newNamespace = SyntaxFactory.NamespaceDeclaration(SyntaxFactory.ParseName(" AppendedNamespaceForConcurrencyTest")) + .WithMembers(compilationUnit.Members) + .WithCloseBraceToken(SyntaxFactory.Token(SyntaxKind.CloseBraceToken).WithLeadingTrivia(SyntaxFactory.Whitespace("\n"))); + if (newNamespace.Members.Any() && newNamespace.Members[0] is NamespaceDeclarationSyntax) + { + // Move the leading trivia of the first member to newNamespace + newNamespace = newNamespace.WithLeadingTrivia(newNamespace.Members[0].GetLeadingTrivia()); + newNamespace = newNamespace.WithMembers(newNamespace.Members.Replace(newNamespace.Members[0], newNamespace.Members[0].WithoutLeadingTrivia())); + } + root = compilationUnit.WithMembers(SyntaxFactory.List(new[] { newNamespace })); + } + + return root.ToFullString(); + } + else + { + return $"namespace AppendedNamespaceForConcurrencyTest {{ {content} {Environment.NewLine}}}"; + } + } + + private string TestCasePath(string fileName) => + Path.GetFullPath(builder.BasePath == null ? Path.Combine(TestCases, fileName) : Path.Combine(TestCases, builder.BasePath, fileName)); + + private void ValidateSingleAnalyzer(string propertyName) + { + if (builder.Analyzers.Length != 1) + { + throw new ArgumentException($"When {propertyName} is set, {nameof(builder.Analyzers)} must contain only 1 analyzer, but {analyzers.Length} were found."); + } + } + + private void ValidateExtension(string path) + { + if (!Path.GetExtension(path).Equals(language.FileExtension, StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException($"Path '{path}' doesn't match {language.LanguageName} file extension '{language.FileExtension}'."); + } + } + + private void ValidateCodeFix() + { + _ = builder.CodeFixedPath ?? throw new ArgumentException($"{nameof(builder.CodeFixedPath)} was not set."); + ValidateSingleAnalyzer(nameof(builder.CodeFix)); + if (builder.Paths.Length != 1) + { + throw new ArgumentException($"{nameof(builder.Paths)} must contain only 1 file, but {builder.Paths.Length} were found."); + } + if (builder.Snippets.Any()) + { + throw new ArgumentException($"{nameof(builder.Snippets)} must be empty when {nameof(builder.CodeFix)} is set."); + } + ValidateExtension(builder.CodeFixedPath); + if (builder.CodeFixedPathBatch is not null) + { + ValidateExtension(builder.CodeFixedPathBatch); + } + if (codeFix.GetType().GetCustomAttribute() is { } codeFixAttribute) + { + if (codeFixAttribute.Languages.Single() != language.LanguageName) + { + throw new ArgumentException($"{analyzers.Single().GetType().Name} language {language.LanguageName} does not match {codeFix.GetType().Name} language."); + } + } + else + { + throw new ArgumentException($"{codeFix.GetType().Name} does not have {nameof(ExportCodeFixProviderAttribute)}."); + } + if (!analyzers.Single().SupportedDiagnostics.Select(x => x.Id).Intersect(codeFix.FixableDiagnosticIds).Any()) + { + throw new ArgumentException($"{analyzers.Single().GetType().Name} does not support diagnostics fixable by the {codeFix.GetType().Name}."); + } + } + } +}