Skip to content

Commit

Permalink
[acceptance-tests] Improve the profiler-stress test runner.
Browse files Browse the repository at this point in the history 
* Emit an NUnit test report file.
* Set a timeout of 6 hours per test.
* Do a thread dump for timed-out tests.
* Don't print stdout/stderr for passing tests.
  • Loading branch information
Alex Rønne Petersen committed Oct 2, 2016
1 parent 77bf968 commit a2f0612
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 18 deletions.
3 changes: 2 additions & 1 deletion acceptance-tests/profiler-stress.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ SYS_REFS = \
System.Data.dll \
System.Runtime.Serialization.dll \
System.Xml.dll \
System.Xml.Linq.dll
System.Xml.Linq.dll \
Mono.Posix.dll

check-profiler-stress:
@$(MAKE) validate-benchmarker RESET_VERSIONS=1
Expand Down
204 changes: 187 additions & 17 deletions acceptance-tests/profiler-stress/runner.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Xml;
using Mono.Unix.Native;
using Newtonsoft.Json;

// Shut up CLS compliance warnings from Json.NET.
Expand All @@ -10,7 +17,8 @@
namespace Mono.Profiling.Tests.Stress {

// https://github.com/xamarin/benchmarker/blob/master/tools/libdbmodel/Benchmark.cs
class Benchmark {
sealed class Benchmark {

public string Name { get; set; }
public string TestDirectory { get; set; }
public bool OnlyExplicit { get; set; }
Expand All @@ -24,8 +32,24 @@ public static Benchmark Load (string file)
}
}

sealed class TestResult {

public Benchmark Benchmark { get; set; }
public ProcessStartInfo StartInfo { get; set; }
public Stopwatch Stopwatch { get; set; } = new Stopwatch ();
public int? ExitCode { get; set; }
public StringBuilder StandardOutput { get; set; } = new StringBuilder ();
public StringBuilder StandardError { get; set; } = new StringBuilder ();
}

static class Program {

static readonly TimeSpan _timeout = TimeSpan.FromHours (6);

static string FilterInvalidXmlChars (string text) {
return Regex.Replace (text, @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]", string.Empty);
}

static int Main ()
{
var depDir = Path.Combine ("..", "external", "benchmarker");
Expand All @@ -44,8 +68,7 @@ static int Main ()
var rand = new Random ();
var cpus = Environment.ProcessorCount;

var successes = 0;
var failures = 0;
var results = new List<TestResult> (benchmarks.Length);

var sw = Stopwatch.StartNew ();

Expand All @@ -66,6 +89,8 @@ static int Main ()
WorkingDirectory = Path.Combine (testDir, bench.TestDirectory),
FileName = monoPath,
Arguments = $"--debug --profile=log:{profOptions} " + string.Join (" ", bench.CommandLine),
RedirectStandardOutput = true,
RedirectStandardError = true,
};

info.EnvironmentVariables.Clear ();
Expand All @@ -77,32 +102,177 @@ static int Main ()
Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} Running {bench.Name} with profiler options: {profOptions}");
Console.ResetColor ();

var sw2 = Stopwatch.StartNew ();
var result = new TestResult {
Benchmark = bench,
StartInfo = info,
};

using (var proc = new Process ()) {
proc.StartInfo = info;

proc.OutputDataReceived += (sender, args) => {
if (args.Data != null)
result.StandardOutput.AppendLine (args.Data);
};

proc.ErrorDataReceived += (sender, args) => {
if (args.Data != null)
result.StandardError.AppendLine (args.Data);
};

result.Stopwatch.Start ();

using (var proc = Process.Start (info)) {
proc.WaitForExit ();
sw2.Stop ();
proc.Start ();

Console.WriteLine ();
proc.BeginOutputReadLine ();
proc.BeginErrorReadLine ();

if (proc.ExitCode != 0)
failures++;
else
successes++;
if (!proc.WaitForExit ((int) _timeout.TotalMilliseconds)) {
// Force a thread dump.
Syscall.kill (proc.Id, Signum.SIGQUIT);
Thread.Sleep (1000);

Console.ForegroundColor = proc.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {sw2.Elapsed.ToString ("G")} and exited with code: {proc.ExitCode}");
try {
proc.Kill ();
} catch (Exception) {
}
} else
result.ExitCode = proc.ExitCode;

result.Stopwatch.Stop ();
}

var resultStr = result.ExitCode == null ? "timed out" : $"exited with code: {result.ExitCode}";

Console.ForegroundColor = result.ExitCode != 0 ? ConsoleColor.Red : ConsoleColor.Green;
Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] {progress} {bench.Name} took {result.Stopwatch.Elapsed.ToString ("G")} and {resultStr}");
Console.ResetColor ();

if (result.ExitCode != 0) {
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine ("===== stdout =====");
Console.ResetColor ();

Console.WriteLine (result.StandardOutput.ToString ());

Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine ("===== stderr =====");
Console.ResetColor ();

Console.WriteLine (result.StandardError.ToString ());
}

results.Add (result);
}

sw.Stop ();

Console.ForegroundColor = failures != 0 ? ConsoleColor.Red : ConsoleColor.Green;
Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{benchmarks.Length} passing tests");
var successes = results.Count (r => r.ExitCode == 0);
var failures = results.Count (r => r.ExitCode != null && r.ExitCode != 0);
var timeouts = results.Count (r => r.ExitCode == null);

var settings = new XmlWriterSettings {
NewLineOnAttributes = true,
Indent = true,
};

using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
writer.WriteStartDocument ();
writer.WriteComment ("This file represents the results of running a test suite");

writer.WriteStartElement ("test-results");
writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
writer.WriteAttributeString ("total", results.Count.ToString ());
writer.WriteAttributeString ("failures", failures.ToString ());
writer.WriteAttributeString ("not-run", "0");
writer.WriteAttributeString ("date", DateTime.Now.ToString ("yyyy-MM-dd"));
writer.WriteAttributeString ("time", DateTime.Now.ToString ("HH:mm:ss"));

writer.WriteStartElement ("environment");
writer.WriteAttributeString ("nunit-version", "2.4.8.0");
writer.WriteAttributeString ("clr-version", Environment.Version.ToString ());
writer.WriteAttributeString ("os-version", Environment.OSVersion.ToString ());
writer.WriteAttributeString ("platform", Environment.OSVersion.Platform.ToString ());
writer.WriteAttributeString ("cwd", Environment.CurrentDirectory);
writer.WriteAttributeString ("machine-name", Environment.MachineName);
writer.WriteAttributeString ("user", Environment.UserName);
writer.WriteAttributeString ("user-domain", Environment.UserDomainName);
writer.WriteEndElement ();

writer.WriteStartElement ("culture-info");
writer.WriteAttributeString ("current-culture", CultureInfo.CurrentCulture.Name);
writer.WriteAttributeString ("current-uiculture", CultureInfo.CurrentUICulture.Name);
writer.WriteEndElement ();

writer.WriteStartElement ("test-suite");
writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
writer.WriteStartElement ("results");

writer.WriteStartElement ("test-suite");
writer.WriteAttributeString ("name", "MonoTests");
writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
writer.WriteStartElement ("results");

writer.WriteStartElement ("test-suite");
writer.WriteAttributeString ("name", "profiler-stress");
writer.WriteAttributeString ("success", (failures + timeouts == 0).ToString ());
writer.WriteAttributeString ("time", ((int) sw.Elapsed.TotalSeconds).ToString ());
writer.WriteAttributeString ("asserts", (failures + timeouts).ToString ());
writer.WriteStartElement ("results");

foreach (var result in results) {
var timeoutStr = result.ExitCode == null ? "_timeout" : string.Empty;

writer.WriteStartElement ("test-case");
writer.WriteAttributeString ("name", $"MonoTests.profiler-stress.{result.Benchmark.Name}{timeoutStr}");
writer.WriteAttributeString ("executed", "True");
writer.WriteAttributeString ("success", (result.ExitCode == 0).ToString ());
writer.WriteAttributeString ("time", ((int) result.Stopwatch.Elapsed.TotalSeconds).ToString ());
writer.WriteAttributeString ("asserts", result.ExitCode == 0 ? "0" : "1");

if (result.ExitCode != 0) {
writer.WriteStartElement ("failure");

writer.WriteStartElement ("message");
writer.WriteCData (FilterInvalidXmlChars (result.StandardOutput.ToString ()));
writer.WriteEndElement ();

writer.WriteStartElement ("stack-trace");
writer.WriteCData (FilterInvalidXmlChars (result.StandardError.ToString ()));
writer.WriteEndElement ();

writer.WriteEndElement ();
}

writer.WriteEndElement ();
}

writer.WriteEndElement ();
writer.WriteEndElement ();

writer.WriteEndElement ();
writer.WriteEndElement ();

writer.WriteEndElement ();
writer.WriteEndElement ();

writer.WriteEndElement ();

writer.WriteEndDocument ();
}

var failureStr = failures + timeouts != 0 ? $" ({failures} failures, {timeouts} timeouts)" : string.Empty;

Console.ForegroundColor = failures + timeouts != 0 ? ConsoleColor.Red : ConsoleColor.Green;
Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Finished with {successes}/{results.Count} passing tests{failureStr}");
Console.ResetColor ();

return failures;
return failures + timeouts;
}
}
}

0 comments on commit a2f0612

Please sign in to comment.