Skip to content

Commit 8153513

Browse files
authored
[Perf, Stress] Case-insensitive command-line parsing (Azure#16965)
- Perf and Stress frameworks should use case-insensitive command-line parsing - Refactor common code into PerfStressUtilities shared source - Add PerfOptions.StatusInterval to align with StressOptions - Fixes Azure#16850
1 parent e357904 commit 8153513

File tree

9 files changed

+235
-240
lines changed

9 files changed

+235
-240
lines changed

common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,8 @@
88
<Reference Include="System.Net.Http" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'" />
99
</ItemGroup>
1010

11+
<ItemGroup>
12+
<Compile Include="..\..\PerfStressShared\PerfStressUtilities.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
13+
</ItemGroup>
14+
1115
</Project>

common/Perf/Azure.Test.Perf/PerfOptions.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ public class PerfOptions
3737
[Option('r', "rate", HelpText = "Target throughput (ops/sec)")]
3838
public int? Rate { get; set; }
3939

40+
[Option("status-interval", Default = 1, HelpText = "Interval to write status to console in seconds")]
41+
public int StatusInterval { get; set; }
42+
4043
[Option("sync", HelpText = "Runs sync version of test")]
4144
public bool Sync { get; set; }
4245

common/Perf/Azure.Test.Perf/PerfProgram.cs

Lines changed: 17 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
using Azure.Test.PerfStress;
45
using CommandLine;
56
using System;
67
using System.Collections.Generic;
78
using System.Diagnostics;
89
using System.Linq;
910
using System.Reflection;
10-
using System.Reflection.Emit;
1111
using System.Runtime;
1212
using System.Text.Json;
1313
using System.Threading;
@@ -31,12 +31,12 @@ public static async Task Main(Assembly assembly, string[] args)
3131

3232
if (testTypes.Any())
3333
{
34-
var optionTypes = GetOptionTypes(testTypes);
35-
await Parser.Default.ParseArguments(args, optionTypes).MapResult<PerfOptions, Task>(
34+
var optionTypes = PerfStressUtilities.GetOptionTypes(testTypes);
35+
await PerfStressUtilities.Parser.ParseArguments(args, optionTypes).MapResult<PerfOptions, Task>(
3636
async o =>
3737
{
3838
var verbName = o.GetType().GetCustomAttribute<VerbAttribute>().Name;
39-
var testType = testTypes.Where(t => GetVerbName(t.Name) == verbName).Single();
39+
var testType = testTypes.Where(t => PerfStressUtilities.GetVerbName(t.Name) == verbName).Single();
4040
await Run(testType, o);
4141
},
4242
errors => Task.CompletedTask
@@ -83,7 +83,7 @@ private static async Task Run(Type testType, PerfOptions options)
8383
Console.WriteLine();
8484

8585
using var setupStatusCts = new CancellationTokenSource();
86-
var setupStatusThread = PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token);
86+
var setupStatusThread = PerfStressUtilities.PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token);
8787

8888
using var cleanupStatusCts = new CancellationTokenSource();
8989
Thread cleanupStatusThread = null;
@@ -109,7 +109,8 @@ private static async Task Run(Type testType, PerfOptions options)
109109

110110
if (options.Warmup > 0)
111111
{
112-
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, "Warmup");
112+
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, options.StatusInterval,
113+
"Warmup");
113114
}
114115

115116
for (var i = 0; i < options.Iterations; i++)
@@ -119,7 +120,8 @@ private static async Task Run(Type testType, PerfOptions options)
119120
{
120121
title += " " + (i + 1);
121122
}
122-
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, title, options.JobStatistics, options.Latency);
123+
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, options.StatusInterval,
124+
title, options.JobStatistics, options.Latency);
123125
}
124126
}
125127
finally
@@ -128,7 +130,7 @@ private static async Task Run(Type testType, PerfOptions options)
128130
{
129131
if (cleanupStatusThread == null)
130132
{
131-
cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
133+
cleanupStatusThread = PerfStressUtilities.PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
132134
}
133135

134136
await Task.WhenAll(tests.Select(t => t.CleanupAsync()));
@@ -141,7 +143,7 @@ private static async Task Run(Type testType, PerfOptions options)
141143
{
142144
if (cleanupStatusThread == null)
143145
{
144-
cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
146+
cleanupStatusThread = PerfStressUtilities.PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
145147
}
146148

147149
await tests[0].GlobalCleanupAsync();
@@ -161,7 +163,7 @@ private static async Task Run(Type testType, PerfOptions options)
161163
}
162164

163165
private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parallel, int? rate,
164-
int durationSeconds, string title, bool jobStatistics = false, bool latency = false)
166+
int durationSeconds, int statusIntervalSeconds, string title, bool jobStatistics = false, bool latency = false)
165167
{
166168
_completedOperations = new int[parallel];
167169
_lastCompletionTimes = new TimeSpan[parallel];
@@ -191,7 +193,7 @@ private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parall
191193
var lastCompleted = 0;
192194

193195
using var progressStatusCts = new CancellationTokenSource();
194-
var progressStatusThread = PrintStatus(
196+
var progressStatusThread = PerfStressUtilities.PrintStatus(
195197
$"=== {title} ===" + Environment.NewLine +
196198
"Current\t\tTotal",
197199
() =>
@@ -202,7 +204,9 @@ private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parall
202204
return currentCompleted + "\t\t" + totalCompleted;
203205
},
204206
newLine: true,
205-
progressStatusCts.Token);
207+
progressStatusCts.Token,
208+
statusIntervalSeconds
209+
);
206210

207211
Thread pendingOperationsThread = null;
208212
if (rate.HasValue)
@@ -375,7 +379,7 @@ private static async Task RunLoopAsync(IPerfTest test, int index, bool latency,
375379
catch (Exception e)
376380
{
377381
// Ignore if any part of the exception chain is type OperationCanceledException
378-
if (!ContainsOperationCanceledException(e))
382+
if (!PerfStressUtilities.ContainsOperationCanceledException(e))
379383
{
380384
throw;
381385
}
@@ -406,112 +410,5 @@ private static Thread WritePendingOperations(int rate, CancellationToken token)
406410

407411
return thread;
408412
}
409-
410-
// Run in dedicated thread instead of using async/await in ThreadPool, to ensure this thread has priority
411-
// and never fails to run to due ThreadPool starvation.
412-
private static Thread PrintStatus(string header, Func<object> status, bool newLine, CancellationToken token)
413-
{
414-
var thread = new Thread(() =>
415-
{
416-
Console.WriteLine(header);
417-
418-
bool needsExtraNewline = false;
419-
420-
while (!token.IsCancellationRequested)
421-
{
422-
try
423-
{
424-
Sleep(TimeSpan.FromSeconds(1), token);
425-
}
426-
catch (OperationCanceledException)
427-
{
428-
}
429-
430-
var obj = status();
431-
432-
if (newLine)
433-
{
434-
Console.WriteLine(obj);
435-
}
436-
else
437-
{
438-
Console.Write(obj);
439-
needsExtraNewline = true;
440-
}
441-
}
442-
443-
if (needsExtraNewline)
444-
{
445-
Console.WriteLine();
446-
}
447-
448-
Console.WriteLine();
449-
});
450-
451-
thread.Start();
452-
453-
return thread;
454-
}
455-
456-
private static void Sleep(TimeSpan timeout, CancellationToken token)
457-
{
458-
var sw = Stopwatch.StartNew();
459-
while (sw.Elapsed < timeout)
460-
{
461-
if (token.IsCancellationRequested)
462-
{
463-
// Simulate behavior of Task.Delay(TimeSpan, CancellationToken)
464-
throw new OperationCanceledException();
465-
}
466-
467-
Thread.Sleep(TimeSpan.FromMilliseconds(10));
468-
}
469-
}
470-
471-
private static bool ContainsOperationCanceledException(Exception e)
472-
{
473-
if (e is OperationCanceledException)
474-
{
475-
return true;
476-
}
477-
else if (e.InnerException != null)
478-
{
479-
return ContainsOperationCanceledException(e.InnerException);
480-
}
481-
else
482-
{
483-
return false;
484-
}
485-
}
486-
487-
// Dynamically create option types with a "Verb" attribute
488-
private static Type[] GetOptionTypes(IEnumerable<Type> testTypes)
489-
{
490-
var optionTypes = new List<Type>();
491-
492-
var ab = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName("Options"), AssemblyBuilderAccess.Run);
493-
var mb = ab.DefineDynamicModule("Options");
494-
495-
foreach (var t in testTypes)
496-
{
497-
var baseOptionsType = t.GetConstructors().First().GetParameters()[0].ParameterType;
498-
var tb = mb.DefineType(t.Name + "Options", TypeAttributes.Public, baseOptionsType);
499-
500-
var attrCtor = typeof(VerbAttribute).GetConstructor(new Type[] { typeof(string), typeof(bool) });
501-
var verbName = GetVerbName(t.Name);
502-
tb.SetCustomAttribute(new CustomAttributeBuilder(attrCtor,
503-
new object[] { verbName, attrCtor.GetParameters()[1].DefaultValue }));
504-
505-
optionTypes.Add(tb.CreateType());
506-
}
507-
508-
return optionTypes.ToArray();
509-
}
510-
511-
private static string GetVerbName(string testName)
512-
{
513-
var lower = testName.ToLowerInvariant();
514-
return lower.EndsWith("test") ? lower.Substring(0, lower.Length - 4) : lower;
515-
}
516413
}
517414
}
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 16
4+
VisualStudioVersion = 16.0.29325.168
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Stress", "..\Stress\Azure.Test.Stress\Azure.Test.Stress.csproj", "{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}"
7+
EndProject
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Stress", "..\Stress\Azure.Sample.Stress\Azure.Sample.Stress.csproj", "{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}"
9+
EndProject
10+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template", "..\..\sdk\template\Azure.Template\src\Azure.Template.csproj", "{6E88344A-4557-424C-9467-9A2AFAD84134}"
11+
EndProject
12+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Stress", "..\..\sdk\template\Azure.Template\stress\Azure.Template.Stress.csproj", "{5B4187EE-9432-4FAD-9266-57D02A6A05F2}"
13+
EndProject
14+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{F4069F59-77C7-4A81-A5CB-D358906B6D5B}"
15+
EndProject
16+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Perf", "..\Perf\Azure.Sample.Perf\Azure.Sample.Perf.csproj", "{29C0605C-5C5D-44A5-8743-BC027D1B9C24}"
17+
EndProject
18+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Perf", "..\..\sdk\template\Azure.Template\perf\Azure.Template.Perf.csproj", "{D4B44894-7439-4242-B94D-D97956EAD938}"
19+
EndProject
20+
Global
21+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
22+
Debug|Any CPU = Debug|Any CPU
23+
Release|Any CPU = Release|Any CPU
24+
EndGlobalSection
25+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
26+
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
27+
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Debug|Any CPU.Build.0 = Debug|Any CPU
28+
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Release|Any CPU.ActiveCfg = Release|Any CPU
29+
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Release|Any CPU.Build.0 = Release|Any CPU
30+
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
31+
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
32+
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
33+
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Release|Any CPU.Build.0 = Release|Any CPU
34+
{6E88344A-4557-424C-9467-9A2AFAD84134}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
35+
{6E88344A-4557-424C-9467-9A2AFAD84134}.Debug|Any CPU.Build.0 = Debug|Any CPU
36+
{6E88344A-4557-424C-9467-9A2AFAD84134}.Release|Any CPU.ActiveCfg = Release|Any CPU
37+
{6E88344A-4557-424C-9467-9A2AFAD84134}.Release|Any CPU.Build.0 = Release|Any CPU
38+
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
39+
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Debug|Any CPU.Build.0 = Debug|Any CPU
40+
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Release|Any CPU.ActiveCfg = Release|Any CPU
41+
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Release|Any CPU.Build.0 = Release|Any CPU
42+
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
43+
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
44+
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
45+
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Release|Any CPU.Build.0 = Release|Any CPU
46+
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
47+
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Debug|Any CPU.Build.0 = Debug|Any CPU
48+
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Release|Any CPU.ActiveCfg = Release|Any CPU
49+
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Release|Any CPU.Build.0 = Release|Any CPU
50+
{D4B44894-7439-4242-B94D-D97956EAD938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
51+
{D4B44894-7439-4242-B94D-D97956EAD938}.Debug|Any CPU.Build.0 = Debug|Any CPU
52+
{D4B44894-7439-4242-B94D-D97956EAD938}.Release|Any CPU.ActiveCfg = Release|Any CPU
53+
{D4B44894-7439-4242-B94D-D97956EAD938}.Release|Any CPU.Build.0 = Release|Any CPU
54+
EndGlobalSection
55+
GlobalSection(SolutionProperties) = preSolution
56+
HideSolutionNode = FALSE
57+
EndGlobalSection
58+
GlobalSection(ExtensibilityGlobals) = postSolution
59+
SolutionGuid = {CD67C1F7-9BEF-4878-A675-898D5D3B48E3}
60+
EndGlobalSection
61+
EndGlobal

0 commit comments

Comments
 (0)