diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ReliabilityFrameworkTestAnalyze.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ReliabilityFrameworkTestAnalyze.Configuration.cs new file mode 100644 index 00000000000..5a15e70fc45 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ReliabilityFrameworkTestAnalyze.Configuration.cs @@ -0,0 +1,64 @@ +using YamlDotNet.Serialization; + +namespace GC.Infrastructure.Core.Configurations.ReliabilityFrameworkTest +{ + public sealed class ReliabilityFrameworkTestAnalyzeConfiguration + { + public required string DebuggerPath { get; set; } + public required List StackFrameKeyWords { get; set; } + public required string CoreRoot { get; set; } + public required string WSLInstanceLocation { get; set; } + public required string DumpFolder { get; set; } + public required string AnalyzeOutputFolder { get; set; } + } + + public static class ReliabilityFrameworkTestAnalyzeConfigurationParser + { + private static readonly IDeserializer _deserializer = + new DeserializerBuilder().IgnoreUnmatchedProperties().Build(); + + public static ReliabilityFrameworkTestAnalyzeConfiguration Parse(string path) + { + // Preconditions. + ConfigurationChecker.VerifyFile(path, nameof(ReliabilityFrameworkTestAnalyzeConfigurationParser)); + + string serializedConfiguration = File.ReadAllText(path); + ReliabilityFrameworkTestAnalyzeConfiguration? configuration = null; + + // This try catch is here because the exception from the YamlDotNet isn't helpful and must be imbued with more details. + try + { + configuration = _deserializer.Deserialize(serializedConfiguration); + } + + catch (Exception ex) + { + throw new ArgumentException($"{nameof(ReliabilityFrameworkTestAnalyzeConfiguration)}: Unable to parse the yaml file because of an error in the syntax. Exception: {ex.Message} \n Call Stack: {ex.StackTrace}"); + } + + if (String.IsNullOrEmpty(configuration.AnalyzeOutputFolder)) + { + throw new ArgumentException($"{nameof(ReliabilityFrameworkTestAnalyzeConfiguration)}: Provide a analyze output folder."); + } + + if (!Path.Exists(configuration.DumpFolder)) + { + throw new ArgumentException($"{nameof(ReliabilityFrameworkTestAnalyzeConfiguration)}: Dump folder doesn't exist."); + } + + // Check Core_Root folder + if (!Path.Exists(configuration.CoreRoot)) + { + throw new ArgumentException($"{nameof(ReliabilityFrameworkTestAnalyzeConfiguration)}: Core_Root doesn't exist."); + } + bool hasCoreRun = Directory.GetFiles(configuration.CoreRoot) + .Any(filePath => Path.GetFileNameWithoutExtension(filePath) == "corerun"); + if (!hasCoreRun) + { + throw new ArgumentException($"{nameof(ReliabilityFrameworkTestAnalyzeConfiguration)}: Provide a valid Core_Root."); + } + + return configuration; + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ReliabilityFrameworkTest/ReliabilityFrameworkTestAggregateCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ReliabilityFrameworkTest/ReliabilityFrameworkTestAggregateCommand.cs new file mode 100644 index 00000000000..069fc21116d --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ReliabilityFrameworkTest/ReliabilityFrameworkTestAggregateCommand.cs @@ -0,0 +1,246 @@ +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Text.RegularExpressions; +using GC.Infrastructure.Core.Configurations; +using GC.Infrastructure.Core.Configurations.ReliabilityFrameworkTest; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace GC.Infrastructure.Commands.ReliabilityFrameworkTest +{ + public class ReliabilityFrameworkTestAggregateCommand : + Command + { + public class ReliabilityFrameworkTestDumpAnalyzeResult + { + public string? AttributedError { get; set; } + public string? DumpName { get; set; } + public string? CallStackForAllThreadsLogName { get; set; } + public string? CallStackLogName { get; set; } + public string? SourceFilePath { get; set; } + public string? LineNumber { get; set; } + } + public sealed class ReliabilityFrameworkTestAggregateSettings : CommandSettings + { + [Description("Path to Configuration.")] + [CommandOption("-c|--configuration")] + public required string ConfigurationPath { get; init; } + } + + public override int Execute([NotNull] CommandContext context, + [NotNull] ReliabilityFrameworkTestAggregateSettings settings) + { + AnsiConsole.Write(new Rule("Aggregate Analysis Results For Reliability Framework Test")); + AnsiConsole.WriteLine(); + + ConfigurationChecker.VerifyFile(settings.ConfigurationPath, + nameof(ReliabilityFrameworkTestAggregateSettings)); + ReliabilityFrameworkTestAnalyzeConfiguration configuration = + ReliabilityFrameworkTestAnalyzeConfigurationParser.Parse(settings.ConfigurationPath); + + AggregateResult(configuration); + return 0; + } + public static void AggregateResult(ReliabilityFrameworkTestAnalyzeConfiguration configuration) + { + List dumpAnalyzeResultList = new List(); + + foreach (string callStackLogPath in Directory.GetFiles(configuration.AnalyzeOutputFolder, "*_callstack.txt")) + { + Console.WriteLine($"====== Extracting information from {callStackLogPath} ======"); + + string dumpPath = callStackLogPath.Replace("_callstack.txt", ".dmp"); + string callStackForAllThreadsLogPath = callStackLogPath.Replace( + "_callstack.txt", "_callstack_allthreads.txt"); + + try + { + string callStack = File.ReadAllText(callStackLogPath); + + // Search for frame contains keywords. + string? frameInfo = FindFrameByKeyWord(configuration.StackFrameKeyWords, callStack); + + // If no line contains given keywords, mark it as unknown error + if (String.IsNullOrEmpty(frameInfo)) + { + ReliabilityFrameworkTestDumpAnalyzeResult unknownErrorResult = new() + { + AttributedError = "Unknown error", + DumpName = Path.GetFileName(dumpPath), + CallStackLogName = Path.GetFileName(callStackLogPath), + CallStackForAllThreadsLogName = Path.GetFileName(callStackForAllThreadsLogPath), + SourceFilePath = String.Empty, + LineNumber = String.Empty + }; + + dumpAnalyzeResultList.Add(unknownErrorResult); + continue; + } + + // Extract source file path and line number + (string, int)? SrcFileLineNumTuple = + ExtractSrcFilePathAndLineNumberFromFrameInfo(frameInfo); + if (!SrcFileLineNumTuple.HasValue) + { + continue; + } + (string srcFilePath, int lineNumber) = SrcFileLineNumTuple.Value; + + int lineIndex = lineNumber - 1; + string realSrcFilePath = string.Empty; + + // Convert source file path if it's in wsl. + if (srcFilePath.StartsWith("/")) + { + if (String.IsNullOrEmpty(configuration.WSLInstanceLocation)) + { + Console.WriteLine($"Console.WriteLine Provide wsl instance location to access source file. "); + continue; + } + + string srcFilePathWithBackSlash = srcFilePath.Replace("/", "\\"); + realSrcFilePath = configuration.WSLInstanceLocation + srcFilePathWithBackSlash; + } + else + { + realSrcFilePath = srcFilePath; + } + + // Get source code line that throw error. + var srcLineList = File.ReadAllLines(realSrcFilePath); + + string srcLine = srcLineList[lineIndex].Trim(); + while (String.IsNullOrEmpty(srcLine) || String.IsNullOrWhiteSpace(srcLine)) + { + lineIndex = lineIndex - 1; + srcLine = srcLineList[lineIndex].Trim(); + } + string error = srcLine; + + ReliabilityFrameworkTestDumpAnalyzeResult dumpAnalyzeResult = new() + { + AttributedError = error, + DumpName = Path.GetFileName(dumpPath), + CallStackLogName = Path.GetFileName(callStackLogPath), + CallStackForAllThreadsLogName = Path.GetFileName(callStackForAllThreadsLogPath), + SourceFilePath = realSrcFilePath, + LineNumber = (lineIndex + 1).ToString() + }; + + dumpAnalyzeResultList.Add(dumpAnalyzeResult); + } + catch (Exception ex) + { + Console.WriteLine($"Console.WriteLine Fail to analyze {callStackLogPath}: {ex.Message}. "); + } + } + + GenerateResultTable(dumpAnalyzeResultList, configuration.AnalyzeOutputFolder); + } + private static void GenerateResultTable(List dumpAnalyzeResultList, + string analyzeOutputFolder) + { + var resultListGroup = dumpAnalyzeResultList.GroupBy(dumpAnalyzeResult => dumpAnalyzeResult.AttributedError); + + StringBuilder sb = new StringBuilder(); + // Write title of table + sb.AppendLine("| Attributed Error | Count/Total(percentage%) | Dump Name | Log Name(Call Stacks of All Threads) | Source File Path | Line Number |"); + sb.AppendLine("| :---------- | :---------: | :---------- | :---------- | :---------- | :---------: |"); + + foreach (IGrouping? group in resultListGroup) + { + var resultListWithoutFirstItem = group.ToList(); + var firstResult = resultListWithoutFirstItem.FirstOrDefault(); + if (firstResult == null) + { + continue; + } + resultListWithoutFirstItem.Remove(firstResult); + + string? attributedError = firstResult.AttributedError; + string proportion = $"{group.Count()}/{dumpAnalyzeResultList.Count}"; + double proportionInPercentage = Convert.ToDouble(group.Count()) / Convert.ToDouble(dumpAnalyzeResultList.Count); + string? dumpName = firstResult.DumpName; + string? callStackForAllThreadsLogName = firstResult.CallStackForAllThreadsLogName; + string? sourceFilePath = firstResult.SourceFilePath; + string? lineNumber = firstResult.LineNumber; + sb.AppendLine($"| {attributedError} | {proportion}({proportionInPercentage * 100}%) | {dumpName} | {callStackForAllThreadsLogName} | {sourceFilePath} | {lineNumber} |"); + + foreach (ReliabilityFrameworkTestDumpAnalyzeResult? dumpAnalyzeResult in resultListWithoutFirstItem) + { + dumpName = dumpAnalyzeResult.DumpName; + callStackForAllThreadsLogName = dumpAnalyzeResult.CallStackForAllThreadsLogName; + sourceFilePath = dumpAnalyzeResult.SourceFilePath; + lineNumber = dumpAnalyzeResult.LineNumber; + sb.AppendLine($"| | | {dumpName} | {callStackForAllThreadsLogName} | {sourceFilePath} | {lineNumber} |"); + } + } + + try + { + string outputPath = Path.Combine(analyzeOutputFolder, "Results.md"); + File.WriteAllText(outputPath, sb.ToString()); + } + catch (Exception ex) + { + throw new Exception($"Fail to write result to markdown: {ex.Message}"); + } + + } + private static (string, int)? ExtractSrcFilePathAndLineNumberFromFrameInfo(string frameInfo) + { + string pattern = @"\[(.*?)\]"; + Match match = Regex.Match(frameInfo, pattern, RegexOptions.Singleline); + + if (!match.Success) + { + Console.WriteLine($"The symbol is not available."); + return null; + } + + string fileNameWithLineNumber = match.Groups[1].Value.Trim(); + string[] splitOutput = fileNameWithLineNumber.Split("@"); + + string? fileName = splitOutput.FirstOrDefault(String.Empty); + if (String.IsNullOrEmpty(fileName)) + { + Console.WriteLine($"Console.WriteLineFail to extract source file path."); + return null; + } + + string? lineNumberstr = splitOutput.LastOrDefault(String.Empty).Trim(); + if (String.IsNullOrEmpty(lineNumberstr)) + { + Console.WriteLine($"Console.WriteLineFail to extract line number."); + return null; + } + + bool success = int.TryParse(lineNumberstr, out int lineNumber); + if (!success) + { + Console.WriteLine($"Console.WriteLineFail to parse line number."); + return null; + } + + return (fileName, lineNumber); + } + private static string? FindFrameByKeyWord(List keyWordList, string callStack) + { + string[] lines = callStack.Split("\n"); + foreach (string line in lines) + { + foreach (string keyWord in keyWordList) + { + if (line.Contains(keyWord)) + { + return line; + } + } + } + + Console.WriteLine($"Console.WriteLineFail to find keyword."); + return null; + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ReliabilityFrameworkTest/ReliabilityFrameworkTestAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ReliabilityFrameworkTest/ReliabilityFrameworkTestAnalyzeCommand.cs new file mode 100644 index 00000000000..28e59ec6f15 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ReliabilityFrameworkTest/ReliabilityFrameworkTestAnalyzeCommand.cs @@ -0,0 +1,169 @@ +using GC.Infrastructure.Core.Configurations; +using GC.Infrastructure.Core.Configurations.ReliabilityFrameworkTest; +using Spectre.Console; +using Spectre.Console.Cli; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text; + + + +namespace GC.Infrastructure.Commands.ReliabilityFrameworkTest +{ + public sealed class ReliabilityFrameworkTestAnalyzeCommand : + Command + { + public class CommandInvokeResult + { + public int ExitCode { get; set; } + public string StdOut { get; set; } + public string StdErr { get; set; } + } + + public sealed class ReliabilityFrameworkTestAnalyzeSettings : CommandSettings + { + [Description("Path to Configuration.")] + [CommandOption("-c|--configuration")] + public required string ConfigurationPath { get; init; } + } + + public override int Execute([NotNull] CommandContext context, + [NotNull] ReliabilityFrameworkTestAnalyzeSettings settings) + { + AnsiConsole.Write(new Rule("Analyze Dumps Collected In Reliability Framework Test")); + AnsiConsole.WriteLine(); + + ConfigurationChecker.VerifyFile(settings.ConfigurationPath, + nameof(ReliabilityFrameworkTestAnalyzeSettings)); + ReliabilityFrameworkTestAnalyzeConfiguration configuration = + ReliabilityFrameworkTestAnalyzeConfigurationParser.Parse(settings.ConfigurationPath); + + Directory.CreateDirectory(configuration.AnalyzeOutputFolder); + + AnalyzeDumps(configuration); + + return 0; + } + + public static void AnalyzeDumps(ReliabilityFrameworkTestAnalyzeConfiguration configuration) + { + foreach (string dumpPath in Directory.GetFiles(configuration.DumpFolder, "*.dmp")) + { + Console.WriteLine($"====== Debugging {dumpPath} ======"); + + string dumpName = Path.GetFileNameWithoutExtension(dumpPath); + string callStackOutputPath = Path.Combine( + configuration.AnalyzeOutputFolder, $"{dumpName}_callstack.txt"); + string callStackForAllThreadsOutputPath = Path.Combine( + configuration.AnalyzeOutputFolder, $"{dumpName}_callstack_allthreads.txt"); + List debugCommandList = new List(){ + $".sympath {configuration.CoreRoot}", + ".reload", + $".logopen {callStackOutputPath}", + "k", + ".logclose", + $".logopen {callStackForAllThreadsOutputPath}", + "~*k", + ".logclose", + }; + + string debuggerScriptPath = Path.Combine(configuration.AnalyzeOutputFolder, "debugging-script.txt"); + GenerateDebuggingScript(debuggerScriptPath, debugCommandList); + DebugDump(new Dictionary(), "", dumpPath, debuggerScriptPath); + } + } + public static void GenerateDebuggingScript(string debuggingScriptPath, List DebugCommandList) + { + // Generate debug script + List exitCommandList; + if (OperatingSystem.IsWindows()) + { + exitCommandList = new() { ".detach", "q" }; + } + else + { + exitCommandList = new() { "exit" }; + } + + List automationDebuggingCommandList = DebugCommandList + .Concat(exitCommandList) + .ToList(); + + File.WriteAllLines(debuggingScriptPath, automationDebuggingCommandList); + } + + public static CommandInvokeResult DebugDump(Dictionary env, + string workingDirectory, + string dumpPath, + string debuggerScriptPath, + bool redirectStdOutErr = true, + bool silent = true) + { + var process = new Process(); + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + + try + { + process.StartInfo.FileName = OperatingSystem.IsWindows() ? "windbgx" : "lldb"; + process.StartInfo.WorkingDirectory = workingDirectory; + process.StartInfo.UseShellExecute = false; + + process.StartInfo.Arguments = OperatingSystem.IsWindows() + ? $"-c $<{debuggerScriptPath} -z {dumpPath}" + : $"-c {dumpPath} -s {debuggerScriptPath} --batch"; + + foreach (var kvp in env) + { + process.StartInfo.EnvironmentVariables[kvp.Key] = kvp.Value; + } + + process.StartInfo.RedirectStandardOutput = redirectStdOutErr; + process.StartInfo.RedirectStandardError = redirectStdOutErr; + + if (redirectStdOutErr) + { + process.OutputDataReceived += (sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + stdout.AppendLine(args.Data); + if (!silent) Console.WriteLine($"STDOUT: {args.Data}"); + } + }; + + process.ErrorDataReceived += (sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + stderr.AppendLine(args.Data); + if (!silent) Console.WriteLine($"STDERR: {args.Data}"); + } + }; + } + + process.Start(); + + if (redirectStdOutErr) + { + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + } + + process.WaitForExit(); + + return new CommandInvokeResult + { + ExitCode = process.ExitCode, + StdOut = stdout.ToString(), + StdErr = stderr.ToString() + }; + } + finally + { + process.Dispose(); + } + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Program.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Program.cs index 0a505869860..16cae413165 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Program.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Program.cs @@ -1,6 +1,7 @@ using GC.Infrastructure.Commands.ASPNetBenchmarks; using GC.Infrastructure.Commands.GCPerfSim; using GC.Infrastructure.Commands.Microbenchmark; +using GC.Infrastructure.Commands.ReliabilityFrameworkTest; using GC.Infrastructure.Commands.RunCommand; using Microsoft.Win32; using Spectre.Console; @@ -59,6 +60,10 @@ internal static void Main(string[] args) // ASP.NET Benchmarks configuration.AddCommand("aspnetbenchmarks"); configuration.AddCommand("aspnetbenchmarks-analyze"); + + // ReliabilityFramework + configuration.AddCommand("rftest-analyze"); + configuration.AddCommand("rftest-aggregate"); }); app.Run(args);