-
Notifications
You must be signed in to change notification settings - Fork 270
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added the Project to Facilitate Notebook Based Testing (#4448)
- Loading branch information
Showing
23 changed files
with
1,758 additions
and
2,842 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
8 changes: 8 additions & 0 deletions
8
...arks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/Exceptions/NotebookException.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
namespace GC.Infrastructure.NotebookTests.Exceptions | ||
{ | ||
public sealed class NotebookExecutionException : Exception | ||
{ | ||
public NotebookExecutionException(string exceptionMessage, string notebookName) | ||
: base($"{notebookName} failed with: {exceptionMessage}") { } | ||
} | ||
} |
9 changes: 9 additions & 0 deletions
9
...astructure/GC.Infrastructure.NotebookTests/Exceptions/NotebookOutputDetectionException.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
namespace GC.Infrastructure.NotebookTests.Exceptions | ||
{ | ||
public sealed class NotebookOutputDetectionException : Exception | ||
{ | ||
public NotebookOutputDetectionException(string notebooks) | ||
: base($"The following notebooks contained outputs that should be cleared before checking in the notebooks: {notebooks}") | ||
{} | ||
} | ||
} |
55 changes: 55 additions & 0 deletions
55
src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/FunctionalTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
using GC.Infrastructure.NotebookTests.Exceptions; | ||
using System.Reflection; | ||
|
||
namespace GC.Infrastructure.NotebookTests | ||
{ | ||
public class FunctionalTests | ||
{ | ||
[SetUp] | ||
public void Setup() | ||
{ | ||
if (!Utils.CheckDotnetReplInstalled()) | ||
{ | ||
Utils.InstallDotnetRepl(); | ||
} | ||
} | ||
|
||
public static IEnumerable<string> GetAllNotebooks() | ||
{ | ||
string notebookPath = Utils.GetNotebookDirectoryPath(); | ||
return Directory.EnumerateFiles(notebookPath, "*.ipynb", SearchOption.AllDirectories); | ||
} | ||
|
||
[Test] | ||
[TestCaseSource(nameof(GetAllNotebooks))] | ||
public void FunctionalTest_RunAllNotebooksToCheckForOutputs_NoOutputsExpected(string notebook) | ||
{ | ||
bool outputDetected = Utils.CheckIfNotebookHasOutputs(notebook); | ||
Assert.False(outputDetected, $"Notebook {notebook} has outputs. No outputs are expected."); | ||
} | ||
|
||
[Test] | ||
[TestCase("GCAnalysisExamples.ipynb")] | ||
[TestCase("CustomDynamicEvents.ipynb")] | ||
[TestCase("CPUAnalysisExamples.ipynb")] | ||
public void FunctionalTest_RunExamples_ExpectsSuccess(string notebookName) | ||
=> Utils.RunNotebookThatsExpectedToPass(Path.Combine(Utils.GetNotebookDirectoryPath(), "Examples", notebookName)); | ||
|
||
[Test] | ||
[TestCase("BenchmarkAnalysis.dib")] | ||
public void FunctionalTest_RunAnalysisNotebooks_ExpectedSuccess(string notebookName) | ||
=> Utils.RunNotebookThatsExpectedToPass(Path.Combine(Utils.GetNotebookDirectoryPath(), notebookName)); | ||
|
||
[Test] | ||
[TestCase("CompilationFailure.dib")] | ||
[TestCase("CompilationFailure.ipynb")] | ||
[TestCase("ExceptionFailure.dib")] | ||
[TestCase("ExceptionFailure.ipynb")] | ||
public void FunctionalTest_IntentionallyFailedNotebook_Failure(string notebookName) | ||
{ | ||
string? executionDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); | ||
string? notebookPath = Path.Combine(executionDirectory!, "TestNotebooks", notebookName); | ||
Utils.RunNotebookThatsExpectedToFail(notebookPath); | ||
} | ||
} | ||
} |
44 changes: 44 additions & 0 deletions
44
.../GC.Infrastructure/GC.Infrastructure.NotebookTests/GC.Infrastructure.NotebookTests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
<Project Sdk="Microsoft.NET.Sdk"> | ||
|
||
<PropertyGroup> | ||
<TargetFramework>net8.0</TargetFramework> | ||
<ImplicitUsings>enable</ImplicitUsings> | ||
<Nullable>enable</Nullable> | ||
|
||
<IsPackable>false</IsPackable> | ||
<IsTestProject>true</IsTestProject> | ||
</PropertyGroup> | ||
|
||
<ItemGroup> | ||
<PackageReference Include="coverlet.collector" Version="6.0.0" /> | ||
<PackageReference Include="FluentAssertions" Version="6.12.0" /> | ||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" /> | ||
<PackageReference Include="NUnit" Version="3.14.0" /> | ||
<PackageReference Include="NUnit.Analyzers" Version="3.9.0" /> | ||
<PackageReference Include="NUnit3TestAdapter" Version="4.5.0" /> | ||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<Using Include="NUnit.Framework" /> | ||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<Folder Include="TestNotebooks\" /> | ||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<None Update="TestNotebooks\CompilationFailure.dib"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</None> | ||
<None Update="TestNotebooks\CompilationFailure.ipynb"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</None> | ||
<None Update="TestNotebooks\ExceptionFailure.dib"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</None> | ||
<None Update="TestNotebooks\ExceptionFailure.ipynb"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</None> | ||
</ItemGroup> | ||
|
||
</Project> |
25 changes: 25 additions & 0 deletions
25
.../gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/GC.Infrastructure.NotebookTests.sln
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
|
||
Microsoft Visual Studio Solution File, Format Version 12.00 | ||
# Visual Studio Version 17 | ||
VisualStudioVersion = 17.5.002.0 | ||
MinimumVisualStudioVersion = 10.0.40219.1 | ||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GC.Infrastructure.NotebookTests", "GC.Infrastructure.NotebookTests.csproj", "{947744DF-0F5D-4640-AC4C-7F5D76E3319F}" | ||
EndProject | ||
Global | ||
GlobalSection(SolutionConfigurationPlatforms) = preSolution | ||
Debug|Any CPU = Debug|Any CPU | ||
Release|Any CPU = Release|Any CPU | ||
EndGlobalSection | ||
GlobalSection(ProjectConfigurationPlatforms) = postSolution | ||
{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
{947744DF-0F5D-4640-AC4C-7F5D76E3319F}.Release|Any CPU.Build.0 = Release|Any CPU | ||
EndGlobalSection | ||
GlobalSection(SolutionProperties) = preSolution | ||
HideSolutionNode = FALSE | ||
EndGlobalSection | ||
GlobalSection(ExtensibilityGlobals) = postSolution | ||
SolutionGuid = {B7770BE1-0EEE-474E-98C7-6B8FF7EC4ED0} | ||
EndGlobalSection | ||
EndGlobal |
152 changes: 152 additions & 0 deletions
152
.../gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/NotebookParser/NotebookDataModel.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,152 @@ | ||
using Newtonsoft.Json; | ||
|
||
namespace GC.Infrastructure.NotebookTests.NotebookParser | ||
{ | ||
// NotebookRoot myDeserializedClass = JsonConvert.DeserializeObject<NotebookRoot>(myJsonResponse); | ||
public class Cell | ||
{ | ||
[JsonProperty("cell_type")] | ||
public required string CellType { get; set; } | ||
|
||
[JsonProperty("execution_count")] | ||
public required object ExecutionCount { get; set; } | ||
|
||
[JsonProperty("metadata")] | ||
public required Metadata Metadata { get; set; } | ||
|
||
[JsonProperty("outputs")] | ||
public List<Output> Outputs { get; } = new List<Output>(); | ||
|
||
[JsonProperty("source")] | ||
public List<string> Source { get; } = new List<string>(); | ||
} | ||
|
||
public class Data | ||
{ | ||
[JsonProperty("text/html")] | ||
public List<string> TextHtml { get; } = new List<string>(); | ||
} | ||
|
||
public class DotnetInteractive | ||
{ | ||
[JsonProperty("language")] | ||
public required string Language { get; set; } | ||
|
||
[JsonProperty("defaultKernelName")] | ||
public required string DefaultKernelName { get; set; } | ||
|
||
[JsonProperty("items")] | ||
public List<Item> Items { get; } = new List<Item>(); | ||
} | ||
|
||
public class Item | ||
{ | ||
[JsonProperty("name")] | ||
public required string Name { get; set; } | ||
} | ||
|
||
public class Kernelspec | ||
{ | ||
[JsonProperty("display_name")] | ||
public required string DisplayName { get; set; } | ||
|
||
[JsonProperty("language")] | ||
public required string Language { get; set; } | ||
|
||
[JsonProperty("name")] | ||
public required string Name { get; set; } | ||
} | ||
|
||
public class LanguageInfo | ||
{ | ||
[JsonProperty("file_extension")] | ||
public required string FileExtension { get; set; } | ||
|
||
[JsonProperty("mimetype")] | ||
public required string Mimetype { get; set; } | ||
|
||
[JsonProperty("name")] | ||
public required string Name { get; set; } | ||
|
||
[JsonProperty("pygments_lexer")] | ||
public required string PygmentsLexer { get; set; } | ||
|
||
[JsonProperty("version")] | ||
public required string Version { get; set; } | ||
} | ||
|
||
public class Metadata | ||
{ | ||
[JsonProperty("dotnet_repl_cellExecutionStartTime")] | ||
public DateTime DotnetReplCellExecutionStartTime { get; set; } | ||
|
||
[JsonProperty("dotnet_repl_cellExecutionEndTime")] | ||
public DateTime DotnetReplCellExecutionEndTime { get; set; } | ||
|
||
[JsonProperty("dotnet_interactive")] | ||
public required DotnetInteractive DotnetInteractive { get; set; } | ||
|
||
[JsonProperty("polyglot_notebook")] | ||
public required PolyglotNotebook PolyglotNotebook { get; set; } | ||
|
||
[JsonProperty("kernelspec")] | ||
public required Kernelspec Kernelspec { get; set; } | ||
|
||
[JsonProperty("language_info")] | ||
public required LanguageInfo LanguageInfo { get; set; } | ||
} | ||
|
||
public class Output | ||
{ | ||
[JsonProperty("name")] | ||
public required string Name { get; set; } | ||
|
||
[JsonProperty("output_type")] | ||
public required string OutputType { get; set; } | ||
|
||
[JsonProperty("text")] | ||
public List<string> Text { get; } = new List<string>(); | ||
|
||
[JsonProperty("data")] | ||
public required Data Data { get; set; } | ||
|
||
[JsonProperty("metadata")] | ||
public required Metadata Metadata { get; set; } | ||
|
||
[JsonProperty("ename")] | ||
public required string Ename { get; set; } | ||
|
||
[JsonProperty("evalue")] | ||
public required string Evalue { get; set; } | ||
|
||
[JsonProperty("traceback")] | ||
public List<string> Traceback { get; } = new List<string>(); | ||
} | ||
|
||
public class PolyglotNotebook | ||
{ | ||
[JsonProperty("kernelName")] | ||
public required string KernelName { get; set; } | ||
|
||
[JsonProperty("defaultKernelName")] | ||
public required string DefaultKernelName { get; set; } | ||
|
||
[JsonProperty("items")] | ||
public List<Item> Items { get; } = new List<Item>(); | ||
} | ||
|
||
public class NotebookRoot | ||
{ | ||
[JsonProperty("cells")] | ||
public List<Cell> Cells { get; } = new List<Cell>(); | ||
|
||
[JsonProperty("metadata")] | ||
public required Metadata Metadata { get; set; } | ||
|
||
[JsonProperty("nbformat")] | ||
public int Nbformat { get; set; } | ||
|
||
[JsonProperty("nbformat_minor")] | ||
public int NbformatMinor { get; set; } | ||
} | ||
} |
64 changes: 64 additions & 0 deletions
64
src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/NotebookRunner.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
using GC.Infrastructure.NotebookTests.NotebookParser; | ||
using Newtonsoft.Json; | ||
using System.Diagnostics; | ||
|
||
namespace GC.Infrastructure.NotebookTests | ||
{ | ||
public sealed class NotebookRunner : IDisposable | ||
{ | ||
internal const int TIMEOUT = 120_000; | ||
private bool disposedValue; | ||
private Process? _dotnetReplProcess; | ||
private readonly string _tempNotebookPath; | ||
|
||
public NotebookRunner(string notebookPath) | ||
{ | ||
NotebookPath = notebookPath; | ||
_dotnetReplProcess = new Process(); | ||
_dotnetReplProcess.StartInfo.FileName = "dotnet-repl"; | ||
_dotnetReplProcess.StartInfo.UseShellExecute = false; | ||
_dotnetReplProcess.StartInfo.Arguments = $"--run {notebookPath} --exit-after-run"; | ||
_tempNotebookPath = Path.GetTempFileName(); | ||
_dotnetReplProcess.StartInfo.Arguments += $" --output-path {_tempNotebookPath}"; | ||
_dotnetReplProcess.StartInfo.WorkingDirectory = Path.GetDirectoryName(notebookPath); | ||
_dotnetReplProcess!.Start(); | ||
_dotnetReplProcess.WaitForExit(TIMEOUT); | ||
|
||
// Assumption: dotnet-repl _always_ creates an output notebook. | ||
NotebookRoot = JsonConvert.DeserializeObject<NotebookRoot>(File.ReadAllText(_tempNotebookPath)); | ||
} | ||
|
||
public string NotebookPath { get; } | ||
public NotebookRoot? NotebookRoot { get; private set; } | ||
public bool? HasExited => _dotnetReplProcess?.HasExited; | ||
public int? ExitCode => _dotnetReplProcess?.ExitCode; | ||
|
||
private void Dispose(bool disposing) | ||
{ | ||
if (!disposedValue) | ||
{ | ||
if (disposing) | ||
{ | ||
NotebookRoot = null; | ||
} | ||
|
||
File.Delete(_tempNotebookPath); | ||
_dotnetReplProcess?.Dispose(); | ||
_dotnetReplProcess = null; | ||
disposedValue = true; | ||
} | ||
} | ||
|
||
~NotebookRunner() | ||
{ | ||
Dispose(disposing: false); | ||
} | ||
|
||
public void Dispose() | ||
{ | ||
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method | ||
Dispose(disposing: true); | ||
System.GC.SuppressFinalize(this); | ||
} | ||
} | ||
} |
8 changes: 8 additions & 0 deletions
8
...gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/TestNotebooks/CompilationFailure.dib
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!meta | ||
|
||
{"kernelInfo":{"defaultKernelName":"csharp","items":[{"aliases":[],"name":"csharp"}]}} | ||
|
||
#!csharp | ||
|
||
int x = | ||
int y = |
Oops, something went wrong.