diff --git a/e2e/stress/IoTClientPerf/IoTClientPerf.csproj b/e2e/stress/IoTClientPerf/IoTClientPerf.csproj index 42a9626332..6b9c603b6f 100644 --- a/e2e/stress/IoTClientPerf/IoTClientPerf.csproj +++ b/e2e/stress/IoTClientPerf/IoTClientPerf.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.1 + netcoreapp2.2 Microsoft.Azure.Devices.E2ETests diff --git a/e2e/stress/IoTClientPerf/PerfTestRunner.cs b/e2e/stress/IoTClientPerf/PerfTestRunner.cs index eba9b2e218..e58dcd1ba4 100644 --- a/e2e/stress/IoTClientPerf/PerfTestRunner.cs +++ b/e2e/stress/IoTClientPerf/PerfTestRunner.cs @@ -25,8 +25,8 @@ public class PerfTestRunner private readonly int _timeSeconds; private readonly Func _scenarioFactory; - private PerfScenario[] _tests; - private Stopwatch _sw = new Stopwatch(); + private readonly PerfScenario[] _tests; + private readonly Stopwatch _sw = new Stopwatch(); public PerfTestRunner( ResultWriter writer, @@ -148,13 +148,13 @@ private async Task LoopAsync() (double avgRps, double stdDevRps) = CalculateAvgAndStDev(statRps); double avgBps = avgRps * _messageSizeBytes; double stdDevBps = stdDevRps * _messageSizeBytes; - SystemMetrics.GetMetrics(out int cpuPercent, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); + SystemMetrics.GetMetrics(out int cpuLoad, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); Console.WriteLine($"[{_sw.Elapsed}] Loop Statistics:"); Console.WriteLine($"RPS : {requestsPerSec,10:N2} R/s Avg: {avgRps,10:N2} R/s +/-StdDev: {stdDevRps,10:N2} R/s"); Console.WriteLine($"Throughput: {GetHumanReadableBytes(transferPerSec)}/s Avg: {GetHumanReadableBytes(avgBps)}/s +/-StdDev: {GetHumanReadableBytes(avgRps)}/s "); Console.WriteLine($"Connected : {devConn,10:N0} "); - Console.WriteLine($"CPU : {cpuPercent,10:N2}% Mem: {GetHumanReadableBytes(memoryBytes)} GC_Mem: {GetHumanReadableBytes(gcBytes)} TCP: {tcpConn,4:N0}"); + Console.WriteLine($"CPU Load : {(float)cpuLoad/100,10:N2} Mem: {GetHumanReadableBytes(memoryBytes)} GC_Mem: {GetHumanReadableBytes(gcBytes)} TCP: {tcpConn,4:N0}"); Console.WriteLine("----"); Console.WriteLine($"TOTALs: "); Console.WriteLine($"Requests : Completed: {statTotalCompleted,10:N0} Faulted: {statTotalFaulted,10:N0} Cancelled: {statTotalCancelled,10:N0}"); @@ -211,12 +211,12 @@ private async Task SetupAllAsync() double totalRequestsPerSec = statTotalCompleted / statTotalSeconds; (double avgRps, double stdDevRps) = CalculateAvgAndStDev(statRps); - SystemMetrics.GetMetrics(out int cpuPercent, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); + SystemMetrics.GetMetrics(out int cpuLoad, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); Console.WriteLine($"[{_sw.Elapsed}] Setup Statistics:"); Console.WriteLine($"RPS : {requestsPerSec,10:N2} R/s Avg: {avgRps,10:N2} R/s +/-StdDev: {stdDevRps,10:N2} R/s"); Console.WriteLine($"Connected : {devConn,10:N0} "); - Console.WriteLine($"CPU : {cpuPercent,10:N2}% Mem: {GetHumanReadableBytes(memoryBytes)} GC_Mem: {GetHumanReadableBytes(gcBytes)} TCP: {tcpConn,4:N0}"); + Console.WriteLine($"CPU Load : {(float)cpuLoad/100,10:N2} Mem: {GetHumanReadableBytes(memoryBytes)} GC_Mem: {GetHumanReadableBytes(gcBytes)} TCP: {tcpConn,4:N0}"); Console.WriteLine("----"); Console.WriteLine($"TOTALs: "); Console.WriteLine($"Requests : Completed: {statTotalCompleted,10:N0} Faulted: {statTotalFaulted,10:N0} Cancelled: {statTotalCancelled,10:N0}"); @@ -257,13 +257,13 @@ private async Task TeardownAllAsync() double totalRequestsPerSec = statTotalCompleted / statTotalSeconds; (double avgRps, double stdDevRps) = CalculateAvgAndStDev(statRps); - SystemMetrics.GetMetrics(out int cpuPercent, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); + SystemMetrics.GetMetrics(out int cpuLoad, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); Console.WriteLine($"[{_sw.Elapsed}] Teardown Statistics:"); Console.WriteLine($"RPS : {requestsPerSec,10:N2} R/s Avg: {avgRps,10:N2} R/s +/-StdDev: {stdDevRps,10:N2} R/s"); Console.WriteLine($"Connected : {devConn,10:N0} "); - Console.WriteLine($"CPU : {cpuPercent,10:N2}% Mem: {GetHumanReadableBytes(memoryBytes)} GC_Mem: {GetHumanReadableBytes(gcBytes)} TCP: {tcpConn,4:N0}"); + Console.WriteLine($"CPU Load : {(float)cpuLoad/100,10:N2} Mem: {GetHumanReadableBytes(memoryBytes)} GC_Mem: {GetHumanReadableBytes(gcBytes)} TCP: {tcpConn,4:N0}"); Console.WriteLine("----"); Console.WriteLine($"TOTALs: "); Console.WriteLine($"Requests : Completed: {statTotalCompleted,10:N0} Faulted: {statTotalFaulted,10:N0} Cancelled: {statTotalCancelled,10:N0}"); diff --git a/e2e/stress/IoTClientPerf/Program.cs b/e2e/stress/IoTClientPerf/Program.cs index 7a570610d2..3116611da9 100644 --- a/e2e/stress/IoTClientPerf/Program.cs +++ b/e2e/stress/IoTClientPerf/Program.cs @@ -3,9 +3,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; -using System.Net.NetworkInformation; namespace Microsoft.Azure.Devices.E2ETests { @@ -49,21 +47,6 @@ private static Dictionary {return new DeviceMethodTest(c);}) }, - { "device_d2c_noretry", - new Tuple>( - "Like device_d2c but will disable retries and create a new DeviceClient when the previous enters a faulted state.", - (c) => {return new DeviceD2CNoRetry(c);})}, - - { "device_c2d_noretry", - new Tuple>( - "Like device_c2d but will disable retries and create a new DeviceClient when the previous enters a faulted state.", - (c) => {return new DeviceC2DNoRetry(c);})}, - - { "device_methods_noretry", - new Tuple>( - "Like device_methods but will disable retries and create a new DeviceClient when the previous enters a faulted state.", - (c) => {return new DeviceMethodsNoRetry(c);})}, - {"service_c2d", new Tuple>( "ServiceClient sending events to devices through IoT Hub.", diff --git a/e2e/stress/IoTClientPerf/Properties/launchSettings.json b/e2e/stress/IoTClientPerf/Properties/launchSettings.json index 56d71f712c..96a7702b47 100644 --- a/e2e/stress/IoTClientPerf/Properties/launchSettings.json +++ b/e2e/stress/IoTClientPerf/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "IoTClientPerf": { "commandName": "Project", - "commandLineArgs": "-o s:\\tmp\\device.csv -t 60 -n 10 -f device_d2c" + "commandLineArgs": "-o device.csv -t 60 -n 10 -f device_d2c" } } } \ No newline at end of file diff --git a/e2e/stress/IoTClientPerf/Reporting/ResultWriter.cs b/e2e/stress/IoTClientPerf/Reporting/ResultWriter.cs index dfce46db5f..a2a52fbe70 100644 --- a/e2e/stress/IoTClientPerf/Reporting/ResultWriter.cs +++ b/e2e/stress/IoTClientPerf/Reporting/ResultWriter.cs @@ -8,12 +8,15 @@ namespace Microsoft.Azure.Devices.E2ETests public abstract class ResultWriter { protected string _header; + private const string NullInstance = "(null)"; public ResultWriter(string header = null) { _header = header; } + public static string IdOf(object value) => value != null ? value.GetType().Name + "#" + GetHashCode(value) : NullInstance; + public Task WriteAsync(TelemetryMetrics m) { return WriteLineAsync(m.ToString()); @@ -22,5 +25,7 @@ public Task WriteAsync(TelemetryMetrics m) public abstract Task FlushAsync(); protected abstract Task WriteLineAsync(string s); + + private static int GetHashCode(object value) => value?.GetHashCode() ?? 0; } } diff --git a/e2e/stress/IoTClientPerf/Reporting/ResultWriterFile.cs b/e2e/stress/IoTClientPerf/Reporting/ResultWriterFile.cs index 9b09d5e0eb..46e6d545c4 100644 --- a/e2e/stress/IoTClientPerf/Reporting/ResultWriterFile.cs +++ b/e2e/stress/IoTClientPerf/Reporting/ResultWriterFile.cs @@ -11,7 +11,7 @@ namespace Microsoft.Azure.Devices.E2ETests { public class ResultWriterFile : ResultWriter { - private const long MaximumFileSize = (long)2 * 1024 * 1024 * 1024; + private const long MaximumFileSize = (long)1 * 1024 * 1024 * 1024; private const int FileBufferBytes = 100 * 1024 * 1024; private StreamWriter _writer; private SemaphoreSlim _semaphore = new SemaphoreSlim(1); diff --git a/e2e/stress/IoTClientPerf/Reporting/SystemMetrics.cs b/e2e/stress/IoTClientPerf/Reporting/SystemMetrics.cs index f3ea358bd0..3e97ed2167 100644 --- a/e2e/stress/IoTClientPerf/Reporting/SystemMetrics.cs +++ b/e2e/stress/IoTClientPerf/Reporting/SystemMetrics.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using System.Net.NetworkInformation; using System.Text; @@ -12,10 +13,10 @@ namespace Microsoft.Azure.Devices.E2ETests { public static class SystemMetrics { - private const int RefreshIntervalMs = 1000; - private static readonly Stopwatch _sw = new Stopwatch(); - private static double s_lastTotalCpuUsageMs = 0.0; - private static int s_cpuPercent; + private const int RefreshIntervalMs = 500; + private static readonly Stopwatch s_sw = new Stopwatch(); + private static TimeSpan s_lastProcCpuUsageMs = TimeSpan.Zero; + private static int s_cpuLoad; private static long s_totalMemoryBytes; private static long s_lastGcBytes; private static long s_lastTcpConnections; @@ -23,12 +24,12 @@ public static class SystemMetrics private static long s_devicesConnected; - private static object s_lock = new object(); + private static readonly object s_lock = new object(); - public static void GetMetrics(out int cpuPercent, out long memoryBytes, out long gcBytes, out long tcpConn, out long devicesConn) + public static void GetMetrics(out int cpuLoad, out long memoryBytes, out long gcBytes, out long tcpConn, out long devicesConn) { EnsureUpToDate(); - cpuPercent = s_cpuPercent; + cpuLoad = s_cpuLoad; memoryBytes = s_totalMemoryBytes; gcBytes = s_lastGcBytes; tcpConn = s_lastTcpConnections; @@ -52,15 +53,23 @@ public static void TcpFilterPort(int port) private static void UpdateCpuUsage() { - var proc = Process.GetCurrentProcess(); - double currentTotalCpuUsageMs = proc.TotalProcessorTime.TotalMilliseconds / Environment.ProcessorCount; - double timeDeltaMs = _sw.Elapsed.TotalMilliseconds; + TimeSpan elapsed = s_sw.Elapsed; + Process proc = Process.GetCurrentProcess(); + + if ((elapsed.Ticks != 0) && (s_lastProcCpuUsageMs != TimeSpan.Zero)) + { - double usedTimeDeltaMs = currentTotalCpuUsageMs - s_lastTotalCpuUsageMs; - if (timeDeltaMs > 0.1) s_cpuPercent = (int)(usedTimeDeltaMs * 100 / timeDeltaMs); - if (s_cpuPercent > 100) s_cpuPercent = 100; + TimeSpan currentTotalCpuUsageMs = proc.TotalProcessorTime; + TimeSpan usedTimeDelta = currentTotalCpuUsageMs - s_lastProcCpuUsageMs; - s_lastTotalCpuUsageMs = currentTotalCpuUsageMs; + s_cpuLoad = (int)(((double)usedTimeDelta.Ticks / elapsed.Ticks) * 100); + } + else + { + s_cpuLoad = -1; + } + + s_lastProcCpuUsageMs = proc.TotalProcessorTime; } private static void UpdateTotalMemoryBytes() @@ -91,16 +100,20 @@ private static void UpdateTCPConnections() private static void EnsureUpToDate() { - if (!_sw.IsRunning || _sw.ElapsedMilliseconds > RefreshIntervalMs) + if (!s_sw.IsRunning) + { + s_sw.Start(); + } + else if (s_sw.ElapsedMilliseconds > RefreshIntervalMs) { lock (s_lock) { - UpdateCpuUsage(); UpdateGCMemoryBytes(); UpdateTCPConnections(); UpdateTotalMemoryBytes(); + UpdateCpuUsage(); - _sw.Restart(); + s_sw.Restart(); } } } diff --git a/e2e/stress/IoTClientPerf/Reporting/TelemetryMetrics.cs b/e2e/stress/IoTClientPerf/Reporting/TelemetryMetrics.cs index 2397de6701..dc35706437 100644 --- a/e2e/stress/IoTClientPerf/Reporting/TelemetryMetrics.cs +++ b/e2e/stress/IoTClientPerf/Reporting/TelemetryMetrics.cs @@ -12,6 +12,7 @@ public class TelemetryMetrics public const string DeviceOperationCreate = "device_create"; public const string DeviceOperationOpen = "device_open"; public const string DeviceOperationClose = "device_close"; + public const string DeviceOperationDispose = "device_dispose"; public const string DeviceOperationSend = "device_send"; public const string DeviceOperationReceive = "device_receive"; public const string DeviceOperationMethodEnable = "device_method_enable"; @@ -30,7 +31,7 @@ public class TelemetryMetrics private static string s_configString; // Contains all Config* parameters. public int? Id; - public string OperationType; // e.g. OpenAsync / SendAsync, etc + private string OperationType; // e.g. OpenAsync / SendAsync, etc public double? ScheduleTime; public double? ExecuteTime; public string ErrorMessage; @@ -62,6 +63,14 @@ public static string GetHeader() "ErrorMessage"; } + public void Clear(string operationType) + { + OperationType = operationType; + ScheduleTime = null; + ExecuteTime = null; + ErrorMessage = null; + } + public static void SetStaticConfigParameters( string runId, int timeSeconds, @@ -87,9 +96,9 @@ public override string ToString() Add(sb, ScheduleTime); Add(sb, ExecuteTime); - SystemMetrics.GetMetrics(out int cpuPercent, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); + SystemMetrics.GetMetrics(out int cpuLoad, out long memoryBytes, out long gcBytes, out long tcpConn, out long devConn); - Add(sb, cpuPercent); + Add(sb, cpuLoad); Add(sb, memoryBytes); Add(sb, gcBytes); Add(sb, tcpConn); diff --git a/e2e/stress/IoTClientPerf/Scenarios/DeviceAllNoRetry.cs b/e2e/stress/IoTClientPerf/Scenarios/DeviceAllNoRetry.cs index 3bb06b2e0b..c062df15dd 100644 --- a/e2e/stress/IoTClientPerf/Scenarios/DeviceAllNoRetry.cs +++ b/e2e/stress/IoTClientPerf/Scenarios/DeviceAllNoRetry.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Azure.Devices.Client.Exceptions; +using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; @@ -8,6 +11,8 @@ namespace Microsoft.Azure.Devices.E2ETests { public class DeviceAllNoRetry : DeviceClientScenario { + private const int DelaySecondsAfterFailure = 1; + private readonly SemaphoreSlim _lock = new SemaphoreSlim(1); private Task _sendTask; private Task _receiveTask; private Task _waitForMethodTask; @@ -28,14 +33,37 @@ public override async Task SetupAsync(CancellationToken ct) public override async Task RunTestAsync(CancellationToken ct) { - SetupTasks(ct); - Task completedTask = await Task.WhenAny(_waitForDisconnectTask, _sendTask, _receiveTask, _waitForMethodTask).ConfigureAwait(false); - - if (completedTask == _waitForDisconnectTask) + try { - DisposeDevice(); - await SetupAsync(ct).ConfigureAwait(false); + await _lock.WaitAsync().ConfigureAwait(false); SetupTasks(ct); + + Task completedTask = await Task.WhenAny(_waitForDisconnectTask, _sendTask, _receiveTask, _waitForMethodTask).ConfigureAwait(false); + + if (completedTask == _waitForDisconnectTask) + { + await DisposeDevice().ConfigureAwait(false); + + try + { + // Drain current operations. Method will not be notified in any way of the disconnect. + await Task.WhenAll(_sendTask, _receiveTask).ConfigureAwait(false); + } + catch (IotHubException) { } + catch (OperationCanceledException) { } + + _waitForDisconnectTask = null; + _sendTask = null; + _receiveTask = null; + _waitForMethodTask = null; + + await Task.Delay(DelaySecondsAfterFailure * 1000).ConfigureAwait(false); + await SetupAsync(ct).ConfigureAwait(false); + } + } + finally + { + _lock.Release(); } } diff --git a/e2e/stress/IoTClientPerf/Scenarios/DeviceC2DNoRetry.cs b/e2e/stress/IoTClientPerf/Scenarios/DeviceC2DNoRetry.cs deleted file mode 100644 index 8fac2e3b14..0000000000 --- a/e2e/stress/IoTClientPerf/Scenarios/DeviceC2DNoRetry.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Azure.Devices.E2ETests -{ - public class DeviceC2DNoRetry : DeviceClientScenario - { - private Task _receiveTask; - private Task _waitForDisconnectTask; - - - public DeviceC2DNoRetry(PerfScenarioConfig config) : base(config) - { - } - - public override async Task SetupAsync(CancellationToken ct) - { - await CreateDeviceAsync().ConfigureAwait(false); - DisableRetry(); - await OpenDeviceAsync(ct).ConfigureAwait(false); - } - - public override async Task RunTestAsync(CancellationToken ct) - { - SetupTasks(ct); - Task completedTask = await Task.WhenAny(_waitForDisconnectTask, _receiveTask).ConfigureAwait(false); - - if (completedTask == _waitForDisconnectTask) - { - DisposeDevice(); - await SetupAsync(ct).ConfigureAwait(false); - SetupTasks(ct); - } - } - - private void SetupTasks(CancellationToken ct) - { - if (_waitForDisconnectTask == null || _waitForDisconnectTask.IsCompleted) _waitForDisconnectTask = WaitForDisconnectedAsync(ct); - if (_receiveTask == null || _receiveTask.IsCompleted) _receiveTask = ReceiveMessageAsync(ct); - } - - public override Task TeardownAsync(CancellationToken ct) - { - return CloseAsync(ct); - } - } -} diff --git a/e2e/stress/IoTClientPerf/Scenarios/DeviceClientScenario.cs b/e2e/stress/IoTClientPerf/Scenarios/DeviceClientScenario.cs index 4e5b136ea6..a2c6fc73e4 100644 --- a/e2e/stress/IoTClientPerf/Scenarios/DeviceClientScenario.cs +++ b/e2e/stress/IoTClientPerf/Scenarios/DeviceClientScenario.cs @@ -17,41 +17,35 @@ public abstract class DeviceClientScenario : PerfScenario private DeviceClient _dc; // Shared by Create, Open and Send - private TelemetryMetrics _m = new TelemetryMetrics(); - private Stopwatch _sw = new Stopwatch(); + private readonly TelemetryMetrics _m = new TelemetryMetrics(); + private readonly Stopwatch _sw = new Stopwatch(); // Separate metrics and time calculation for operations that can be parallelized. - private TelemetryMetrics _mRecv = new TelemetryMetrics(); - private Stopwatch _swRecv = new Stopwatch(); + private readonly TelemetryMetrics _mRecv = new TelemetryMetrics(); + private readonly Stopwatch _swRecv = new Stopwatch(); - private const string TestMethodName = "PerfTestMethod"; - private TelemetryMetrics _mMethod = new TelemetryMetrics(); - private Stopwatch _swMethod = new Stopwatch(); - private SemaphoreSlim _methodSemaphore = new SemaphoreSlim(0); + private const string TestMethodName = "SendMessageToDevice"; + private readonly TelemetryMetrics _mMethod = new TelemetryMetrics(); + private readonly Stopwatch _swMethod = new Stopwatch(); + private readonly SemaphoreSlim _methodSemaphore = new SemaphoreSlim(0); private static readonly MethodResponse s_methodResponse = new MethodResponse(200); - private TelemetryMetrics _mConnectionStatus = new TelemetryMetrics(); - private SemaphoreSlim _connectionStatusChangedSemaphore = new SemaphoreSlim(0); - private SemaphoreSlim _waitForDisconnectSemaphore = new SemaphoreSlim(0); + private readonly TelemetryMetrics _mConnectionStatus = new TelemetryMetrics(); + private readonly SemaphoreSlim _connectionStatusChangedSemaphore = new SemaphoreSlim(1); + private readonly SemaphoreSlim _waitForDisconnectSemaphore = new SemaphoreSlim(0, 1); private bool _connected; - private byte[] _messageBytes; + private readonly byte[] _messageBytes; - private bool _pooled; - private int _poolSize; + private readonly bool _pooled; + private readonly int _poolSize; public DeviceClientScenario(PerfScenarioConfig config) : base(config) { _m.Id = _id; - _mRecv.Id = _id; - _mRecv.OperationType = TelemetryMetrics.DeviceOperationReceive; - _mMethod.Id = _id; - _mConnectionStatus.Id = _id; - _mConnectionStatus.ExecuteTime = null; - _mConnectionStatus.ScheduleTime = null; _messageBytes = new byte[_sizeBytes]; @@ -64,7 +58,7 @@ public DeviceClientScenario(PerfScenarioConfig config) : base(config) protected async Task CreateDeviceAsync() { _sw.Restart(); - _m.OperationType = TelemetryMetrics.DeviceOperationCreate; + _m.Clear(TelemetryMetrics.DeviceOperationCreate); ITransportSettings transportSettings = null; @@ -108,7 +102,6 @@ protected async Task CreateDeviceAsync() _dc.SetConnectionStatusChangesHandler(OnConnectionStatusChanged); _m.ExecuteTime = _sw.ElapsedMilliseconds; - _m.ScheduleTime = null; // sync operation await _writer.WriteAsync(_m).ConfigureAwait(false); } @@ -132,8 +125,13 @@ private async void OnConnectionStatusChanged(ConnectionStatus status, Connection _connected = false; } - _mConnectionStatus.OperationType = TelemetryMetrics.DeviceStateDisconnected; - _waitForDisconnectSemaphore.Release(); + _mConnectionStatus.Clear(TelemetryMetrics.DeviceStateDisconnected); + try + { + _waitForDisconnectSemaphore.Release(); + } + catch (SemaphoreFullException) { } + break; case ConnectionStatus.Connected: if (!_connected) @@ -142,7 +140,7 @@ private async void OnConnectionStatusChanged(ConnectionStatus status, Connection _connected = true; } - _mConnectionStatus.OperationType = TelemetryMetrics.DeviceStateConnected; + _mConnectionStatus.Clear(TelemetryMetrics.DeviceStateConnected); break; case ConnectionStatus.Disconnected_Retrying: if (_connected) @@ -151,8 +149,14 @@ private async void OnConnectionStatusChanged(ConnectionStatus status, Connection _connected = false; } - _mConnectionStatus.OperationType = TelemetryMetrics.DeviceStateDisconnectedRetrying; - _waitForDisconnectSemaphore.Release(); + _mConnectionStatus.Clear(TelemetryMetrics.DeviceStateDisconnectedRetrying); + + try + { + _waitForDisconnectSemaphore.Release(); + } + catch (SemaphoreFullException) { } + break; case ConnectionStatus.Disabled: if (_connected) @@ -161,14 +165,14 @@ private async void OnConnectionStatusChanged(ConnectionStatus status, Connection _connected = false; } - _mConnectionStatus.OperationType = TelemetryMetrics.DeviceStateDisconnected; + _mConnectionStatus.Clear(TelemetryMetrics.DeviceStateDisconnected); break; default: - _mConnectionStatus.OperationType = TelemetryMetrics.DeviceStateUnknown; + _mConnectionStatus.Clear(TelemetryMetrics.DeviceStateUnknown); break; } - _mConnectionStatus.ErrorMessage = $"ConnectionStatus: {status} reason: {reason}"; + _mConnectionStatus.ErrorMessage = $"ConnectionStatus: {status} reason: {reason} id: {ResultWriter.IdOf(_dc)}"; await _writer.WriteAsync(_mConnectionStatus).ConfigureAwait(false); } finally @@ -184,8 +188,8 @@ protected Task WaitForDisconnectedAsync(CancellationToken ct) protected async Task OpenDeviceAsync(CancellationToken ct) { - _m.OperationType = TelemetryMetrics.DeviceOperationOpen; - _m.ScheduleTime = null; + _m.Clear(TelemetryMetrics.DeviceOperationOpen); + _sw.Restart(); try { @@ -197,7 +201,7 @@ protected async Task OpenDeviceAsync(CancellationToken ct) } catch (Exception ex) { - _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + SetErrorMessage(_m, ex); throw; } finally @@ -209,8 +213,8 @@ protected async Task OpenDeviceAsync(CancellationToken ct) protected async Task SendMessageAsync(CancellationToken ct) { - _m.OperationType = TelemetryMetrics.DeviceOperationSend; - _m.ScheduleTime = null; + _m.Clear(TelemetryMetrics.DeviceOperationSend); + _sw.Restart(); try @@ -224,7 +228,7 @@ protected async Task SendMessageAsync(CancellationToken ct) } catch (Exception ex) { - _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + SetErrorMessage(_m, ex); throw; } finally @@ -236,7 +240,7 @@ protected async Task SendMessageAsync(CancellationToken ct) protected async Task ReceiveMessageAsync(CancellationToken ct) { - _mRecv.ScheduleTime = null; + _mRecv.Clear(TelemetryMetrics.DeviceOperationReceive); _swRecv.Restart(); try @@ -253,7 +257,7 @@ protected async Task ReceiveMessageAsync(CancellationToken ct) } catch (Exception ex) { - _mRecv.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + SetErrorMessage(_mRecv, ex); throw; } finally @@ -265,8 +269,7 @@ protected async Task ReceiveMessageAsync(CancellationToken ct) protected async Task EnableMethodsAsync(CancellationToken ct) { - _mMethod.ScheduleTime = null; - _mMethod.OperationType = TelemetryMetrics.DeviceOperationMethodEnable; + _mMethod.Clear(TelemetryMetrics.DeviceOperationMethodEnable); _swMethod.Restart(); try @@ -279,7 +282,7 @@ protected async Task EnableMethodsAsync(CancellationToken ct) } catch (Exception ex) { - _mMethod.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + SetErrorMessage(_mMethod, ex); throw; } finally @@ -297,8 +300,7 @@ private Task MethodHandlerAsync(MethodRequest methodRequest, obj protected async Task WaitForMethodAsync(CancellationToken ct) { - _mMethod.ScheduleTime = null; - _mMethod.OperationType = TelemetryMetrics.DeviceOperationMethodCalled; + _mMethod.Clear(TelemetryMetrics.DeviceOperationMethodCalled); _swMethod.Restart(); try @@ -311,7 +313,7 @@ protected async Task WaitForMethodAsync(CancellationToken ct) } catch (Exception ex) { - _mMethod.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + SetErrorMessage(_mMethod, ex); throw; } finally @@ -325,8 +327,7 @@ protected async Task CloseAsync(CancellationToken ct) { if (_dc == null) return; - _m.ScheduleTime = null; - _m.OperationType = TelemetryMetrics.DeviceOperationClose; + _m.Clear(TelemetryMetrics.DeviceOperationClose); _sw.Restart(); try @@ -335,7 +336,7 @@ protected async Task CloseAsync(CancellationToken ct) } catch (Exception ex) { - _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + SetErrorMessage(_m, ex); throw; } finally @@ -345,9 +346,16 @@ protected async Task CloseAsync(CancellationToken ct) } } - protected void DisposeDevice() + private void SetErrorMessage(TelemetryMetrics m, Exception ex) { + m.ErrorMessage = $"{ex.GetType().Name} id: {ResultWriter.IdOf(_dc)} - {ex.Message}"; + } + + protected async Task DisposeDevice() + { + _m.Clear(TelemetryMetrics.DeviceOperationDispose); _dc.Dispose(); + await _writer.WriteAsync(_m).ConfigureAwait(false); } } } diff --git a/e2e/stress/IoTClientPerf/Scenarios/DeviceD2CNoRetry.cs b/e2e/stress/IoTClientPerf/Scenarios/DeviceD2CNoRetry.cs deleted file mode 100644 index 258b900113..0000000000 --- a/e2e/stress/IoTClientPerf/Scenarios/DeviceD2CNoRetry.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Azure.Devices.E2ETests -{ - public class DeviceD2CNoRetry : DeviceClientScenario - { - private Task _sendTask; - private Task _waitForDisconnectTask; - - - public DeviceD2CNoRetry(PerfScenarioConfig config) : base(config) - { - } - - public override async Task SetupAsync(CancellationToken ct) - { - await CreateDeviceAsync().ConfigureAwait(false); - DisableRetry(); - await OpenDeviceAsync(ct).ConfigureAwait(false); - } - - public override async Task RunTestAsync(CancellationToken ct) - { - SetupTasks(ct); - Task completedTask = await Task.WhenAny(_waitForDisconnectTask, _sendTask).ConfigureAwait(false); - - if (completedTask == _waitForDisconnectTask) - { - DisposeDevice(); - await SetupAsync(ct).ConfigureAwait(false); - SetupTasks(ct); - } - } - - private void SetupTasks(CancellationToken ct) - { - if (_waitForDisconnectTask == null || _waitForDisconnectTask.IsCompleted) _waitForDisconnectTask = WaitForDisconnectedAsync(ct); - if (_sendTask == null || _sendTask.IsCompleted) _sendTask = SendMessageAsync(ct); - } - - public override Task TeardownAsync(CancellationToken ct) - { - return CloseAsync(ct); - } - } -} diff --git a/e2e/stress/IoTClientPerf/Scenarios/DeviceMethodsNoRetry.cs b/e2e/stress/IoTClientPerf/Scenarios/DeviceMethodsNoRetry.cs deleted file mode 100644 index 56ae25dbd8..0000000000 --- a/e2e/stress/IoTClientPerf/Scenarios/DeviceMethodsNoRetry.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Azure.Devices.E2ETests -{ - public class DeviceMethodsNoRetry : DeviceClientScenario - { - private Task _waitForMethodTask; - private Task _waitForDisconnectTask; - - public DeviceMethodsNoRetry(PerfScenarioConfig config) : base(config) - { - } - - public override async Task SetupAsync(CancellationToken ct) - { - await CreateDeviceAsync().ConfigureAwait(false); - DisableRetry(); - await OpenDeviceAsync(ct).ConfigureAwait(false); - await EnableMethodsAsync(ct).ConfigureAwait(false); - } - - public override async Task RunTestAsync(CancellationToken ct) - { - SetupTasks(ct); - Task completedTask = await Task.WhenAny(_waitForDisconnectTask, _waitForMethodTask).ConfigureAwait(false); - - if (completedTask == _waitForDisconnectTask) - { - DisposeDevice(); - await SetupAsync(ct).ConfigureAwait(false); - SetupTasks(ct); - } - } - - private void SetupTasks(CancellationToken ct) - { - if (_waitForDisconnectTask == null || _waitForDisconnectTask.IsCompleted) _waitForDisconnectTask = WaitForDisconnectedAsync(ct); - if (_waitForMethodTask == null || _waitForMethodTask.IsCompleted) _waitForMethodTask = WaitForMethodAsync(ct); - } - - public override Task TeardownAsync(CancellationToken ct) - { - return CloseAsync(ct); - } - } -} diff --git a/e2e/stress/IoTClientPerf/Scenarios/DeviceOneD2CTest.cs b/e2e/stress/IoTClientPerf/Scenarios/DeviceOneD2CTest.cs index 030f039993..ab5ff708cf 100644 --- a/e2e/stress/IoTClientPerf/Scenarios/DeviceOneD2CTest.cs +++ b/e2e/stress/IoTClientPerf/Scenarios/DeviceOneD2CTest.cs @@ -42,7 +42,7 @@ public override async Task SetupAsync(CancellationToken ct) private async Task CreateDeviceAsync() { _sw.Restart(); - _m.OperationType = TelemetryMetrics.DeviceOperationCreate; + _m.Clear(TelemetryMetrics.DeviceOperationCreate); ITransportSettings transportSettings = null; @@ -79,8 +79,7 @@ private async Task CreateDeviceAsync() protected async Task OpenDeviceAsync(CancellationToken ct) { ExceptionDispatchInfo exInfo = null; - _m.OperationType = TelemetryMetrics.DeviceOperationOpen; - _m.ScheduleTime = null; + _m.Clear(TelemetryMetrics.DeviceOperationOpen); _sw.Restart(); try { @@ -110,8 +109,7 @@ public override Task RunTestAsync(CancellationToken ct) protected async Task SendMessageAsync(CancellationToken ct) { ExceptionDispatchInfo exInfo = null; - _m.OperationType = TelemetryMetrics.DeviceOperationSend; - _m.ScheduleTime = null; + _m.Clear(TelemetryMetrics.DeviceOperationSend); _sw.Restart(); try diff --git a/e2e/stress/IoTClientPerf/Scenarios/HarnessBaseline.cs b/e2e/stress/IoTClientPerf/Scenarios/HarnessBaseline.cs index 4fbf7ccf8c..d387028dcc 100644 --- a/e2e/stress/IoTClientPerf/Scenarios/HarnessBaseline.cs +++ b/e2e/stress/IoTClientPerf/Scenarios/HarnessBaseline.cs @@ -24,7 +24,7 @@ public HarnessBaseline(PerfScenarioConfig config) : base(config) public override async Task SetupAsync(CancellationToken ct) { _sw.Restart(); - _m.OperationType = "baseline_setup"; + _m.Clear("baseline_setup"); await Task.Delay(100).ConfigureAwait(false); _m.ExecuteTime = _sw.ElapsedMilliseconds; _m.ScheduleTime = null; // sync operation @@ -34,7 +34,7 @@ public override async Task SetupAsync(CancellationToken ct) public override async Task RunTestAsync(CancellationToken ct) { _sw.Restart(); - _m.OperationType = "baseline_run"; + _m.Clear("baseline_run"); await Task.Delay(100).ConfigureAwait(false); _m.ExecuteTime = _sw.ElapsedMilliseconds; _m.ScheduleTime = null; // sync operation @@ -44,7 +44,7 @@ public override async Task RunTestAsync(CancellationToken ct) public override async Task TeardownAsync(CancellationToken ct) { _sw.Restart(); - _m.OperationType = "baseline_teardown"; + _m.Clear("baseline_teardown"); await Task.Delay(100).ConfigureAwait(false); _m.ExecuteTime = _sw.ElapsedMilliseconds; _m.ScheduleTime = null; // sync operation diff --git a/e2e/stress/IoTClientPerf/Scenarios/ServiceClientScenarios.cs b/e2e/stress/IoTClientPerf/Scenarios/ServiceClientScenarios.cs index ef1d586d32..443c6c303f 100644 --- a/e2e/stress/IoTClientPerf/Scenarios/ServiceClientScenarios.cs +++ b/e2e/stress/IoTClientPerf/Scenarios/ServiceClientScenarios.cs @@ -3,7 +3,7 @@ using System; using System.Diagnostics; -using System.Runtime.ExceptionServices; +using System.Globalization; using System.Threading; using System.Threading.Tasks; @@ -20,6 +20,11 @@ public abstract class ServiceClientScenario : PerfScenario // Separate metrics and time calculation for operations that can be parallelized. private const string TestMethodName = "PerfTestMethod"; private const int MethodPassStatus = 200; + private const int MethodConnectionTimeoutSeconds = 30; + private const int MethodResponseTimeoutSeconds = 30; + + private const int C2DExpiryTimeSeconds = 90; + private TelemetryMetrics _mMethod = new TelemetryMetrics(); private Stopwatch _swMethod = new Stopwatch(); @@ -42,13 +47,32 @@ public ServiceClientScenario(PerfScenarioConfig config) : base(config) protected void CreateServiceClient() { - if (_id == 0) s_sc = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); + if (_id != 0) return; + s_sc?.Dispose(); + + switch (_transport) + { + case Client.TransportType.Amqp_WebSocket_Only: + s_sc = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString, TransportType.Amqp_WebSocket_Only); + break; + case Client.TransportType.Amqp_Tcp_Only: + s_sc = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString, TransportType.Amqp); + break; + + case Client.TransportType.Amqp: + case Client.TransportType.Http1: + case Client.TransportType.Mqtt: + case Client.TransportType.Mqtt_WebSocket_Only: + case Client.TransportType.Mqtt_Tcp_Only: + default: + s_sc = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString); + break; + } } protected async Task OpenServiceClientAsync(CancellationToken ct) { - _m.OperationType = TelemetryMetrics.ServiceOperationOpen; - _m.ScheduleTime = null; + _m.Clear(TelemetryMetrics.ServiceOperationOpen); _sw.Restart(); try { @@ -58,6 +82,12 @@ protected async Task OpenServiceClientAsync(CancellationToken ct) _sw.Restart(); await t.ConfigureAwait(false); } + catch (NullReferenceException ex) // TODO #708 - ServiceClient AMQP will continuously fail with NullRefException after fault. + { + CreateServiceClient(); + _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + throw; + } catch (Exception ex) { _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; @@ -72,19 +102,25 @@ protected async Task OpenServiceClientAsync(CancellationToken ct) protected async Task SendMessageAsync(CancellationToken ct) { - _m.OperationType = TelemetryMetrics.ServiceOperationSend; - _m.ScheduleTime = null; + _m.Clear(TelemetryMetrics.ServiceOperationSend); _sw.Restart(); try { var message = new Message(_messageBytes); + message.ExpiryTimeUtc = DateTime.UtcNow + TimeSpan.FromSeconds(C2DExpiryTimeSeconds); Task t = s_sc.SendAsync(Configuration.Stress.GetDeviceNameById(_id, _authType), message); _m.ScheduleTime = _sw.ElapsedMilliseconds; _sw.Restart(); await t.ConfigureAwait(false); } + catch (NullReferenceException ex) // TODO #708 - ServiceClient AMQP will continuously fail with NullRefException after fault. + { + CreateServiceClient(); + _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; + throw; + } catch (Exception ex) { _m.ErrorMessage = $"{ex.GetType().Name} - {ex.Message}"; @@ -99,15 +135,17 @@ protected async Task SendMessageAsync(CancellationToken ct) protected async Task CallMethodAsync(CancellationToken ct) { - _mMethod.ScheduleTime = null; - _mMethod.OperationType = TelemetryMetrics.ServiceOperationMethodCall; + _mMethod.Clear(TelemetryMetrics.ServiceOperationMethodCall); _swMethod.Restart(); try { string deviceId = Configuration.Stress.GetDeviceNameById(_id, _authType); - var methodCall = new CloudToDeviceMethod(TestMethodName); + var methodCall = new CloudToDeviceMethod( + methodName: TestMethodName, + responseTimeout: TimeSpan.FromSeconds(MethodResponseTimeoutSeconds), + connectionTimeout: TimeSpan.FromSeconds(MethodConnectionTimeoutSeconds)); methodCall.SetPayloadJson(_methodPayload); Task t = s_sc.InvokeDeviceMethodAsync(Configuration.Stress.GetDeviceNameById(_id, _authType), methodCall); _mMethod.ScheduleTime = _swMethod.ElapsedMilliseconds; @@ -137,8 +175,7 @@ protected async Task CloseAsync(CancellationToken ct) { if (s_sc == null) return; - _m.ScheduleTime = null; - _m.OperationType = TelemetryMetrics.ServiceOperationClose; + _m.Clear(TelemetryMetrics.ServiceOperationClose); _sw.Restart(); try diff --git a/e2e/stress/IoTClientPerf/scripts/runClient.ps1 b/e2e/stress/IoTClientPerf/scripts/runClient.ps1 index 7370801637..b31fd4dc93 100644 --- a/e2e/stress/IoTClientPerf/scripts/runClient.ps1 +++ b/e2e/stress/IoTClientPerf/scripts/runClient.ps1 @@ -24,18 +24,31 @@ Param( $host.ui.RawUI.WindowTitle = "Azure IoT SDK: Device Stress" +$fileName = [io.path]::GetFileNameWithoutExtension($outputFile) +$filePath = [io.path]::GetDirectoryName($outputFile) +if ($filePath -eq "") { + $filePath = pwd +} Write-Host -ForegroundColor Cyan "`nDEVICE: C2D`n" -& dotnet run --no-build -c Release -- -t $durationSeconds -o $outputFile -p $protocol -n $clients -c $connections -f device_c2d +$scenario = "device_c2d" +$out = Join-Path $filePath "$fileName.$($scenario).csv" +& dotnet run --no-build -c Release -- -t $durationSeconds -o $out -p $protocol -n $clients -c $connections -f $scenario -s 2048 Write-Host -ForegroundColor Cyan "`nDEVICE: Methods`n" -& dotnet run --no-build -c Release -- -t $durationSeconds -o $outputFile -p $protocol -n $clients -c $connections -f device_method - +$scenario = "device_method" +$out = Join-Path $filePath "$fileName.$($scenario).csv" +& dotnet run --no-build -c Release -- -t $durationSeconds -o $out -p $protocol -n $clients -c $connections -f $scenario -s 2048 Write-Host -ForegroundColor Cyan "`nDEVICE: All`n" -& dotnet run --no-build -c Release -- -t $durationSeconds -o $outputFile -p $protocol -n $clients -c $connections -f device_all +$scenario = "device_all" +$out = Join-Path $filePath "$fileName.$($scenario).csv" + +& dotnet run --no-build -c Release -- -t $durationSeconds -o $out -p $protocol -n $clients -c $connections -f $scenario -s 2048 Write-Host -ForegroundColor Cyan "`nDEVICE: D2C`n" -& dotnet run --no-build -c Release -- -t $durationSeconds -o $outputFile -p $protocol -n $clients -c $connections -f device_d2c +$scenario = "device_d2c" +$out = Join-Path $filePath "$fileName.$($scenario).csv" +& dotnet run --no-build -c Release -- -t $durationSeconds -o $out -p $protocol -n $clients -c $connections -f $scenario -s 2048 diff --git a/e2e/stress/IoTClientPerf/scripts/runServer.ps1 b/e2e/stress/IoTClientPerf/scripts/runServer.ps1 index 39a5ba28b2..e38eb0edfd 100644 --- a/e2e/stress/IoTClientPerf/scripts/runServer.ps1 +++ b/e2e/stress/IoTClientPerf/scripts/runServer.ps1 @@ -19,13 +19,16 @@ Param( $protocol = "amqp", $connections = 10, $outputFile = "service.csv", - $durationSeconds = 1800, + $durationSeconds = 300, $type = $null ) $fileName = [io.path]::GetFileNameWithoutExtension($outputFile) $filePath = [io.path]::GetDirectoryName($outputFile) +if ($filePath -eq "") { + $filePath = pwd +} if ($type -eq $null) { @@ -44,15 +47,25 @@ elseif ($type -eq "methods") $host.ui.RawUI.WindowTitle = "Azure IoT SDK: Service Stress [Methods]" Write-Host -ForegroundColor Cyan "`nSERVICE: Methods`n" - $out = Join-Path $filePath "$fileName.method.csv" $scenario = "service_method" + $out = Join-Path $filePath "$fileName.$($scenario).csv" + if (Test-Path $out) + { + rm $out + } + } elseif ($type -eq "c2d") { $host.ui.RawUI.WindowTitle = "Azure IoT SDK: Service Stress [C2D]" Write-Host -ForegroundColor Cyan "`nSERVICE: C2D`n" - $out = Join-Path $filePath "$fileName.c2d.csv" $scenario = "service_c2d" + $out = Join-Path $filePath "$fileName.$($scenario).csv" + + if (Test-Path $out) + { + rm $out + } } else { @@ -60,5 +73,5 @@ else } -& dotnet run --no-build -c Release -- -t $durationSeconds -o $out -p $protocol -n $clients -c $connections -f $scenario +& dotnet run --no-build -c Release -- -t $durationSeconds -o $out -p $protocol -n $clients -c $connections -f $scenario -s 2048 Read-Host "Press ENTER to close"