First pass at tooling to allow examples to be tested

standard/classes.md

@@ -68,6 +68,7 @@ When a non-abstract class is derived from an abstract class, the non-abstract cl
 
 > *Example*: In the following code
 >
+> <!-- Example: {template:"standalone-lib", name:"AbstractMethodImplementation"} -->
 > ```csharp
 > abstract class A
 > {
@@ -186,6 +187,7 @@ When a *class_type* is included in the *class_base*, it specifies the direct bas
 
 > *Example*: In the following code
 >
+> <!-- Example: {template:"standalone-lib", name:"DirectBaseClass"} -->
 > ```csharp
 > class A {}
 > class B : A {}
@@ -199,6 +201,7 @@ For a constructed class type, including a nested type declared within a generic
 
 > *Example*: Given the generic class declarations
 >
+> <!-- Example: {template:"standalone-lib", name:"GenericBaseClass", replaceEllipsis:true} -->
 > ```csharp
 > class B<U,V> {...}
 > class G<T> : B<string,T[]> {...}
@@ -212,17 +215,19 @@ The base class specified in a class declaration can be a constructed class type
 
 > *Example*:
 >
+> <!-- TODO: This example has been modified to add 1, 2, 3 to the class names. Is that okay? -->
+> <!-- Example: {template:"standalone-lib", name:"TypeParameterUsedAsBaseClass", expectedErrors:["CS0689"]} -->
 > ```csharp
 > class Base<T> {}
 >
 > // Valid, non-constructed class with constructed base class
-> class Extend : Base<int>
+> class Extend1 : Base<int> {}
 >
 > // Error, type parameter used as base class
-> class Extend<V> : V {}
+> class Extend2<V> : V {}
 >
 > // Valid, type parameter used as type argument for base class
-> class Extend<V> : Base<V> {}
+> class Extend3<V> : Base<V> {}
 > ```
 >
 > *end example*
@@ -235,6 +240,7 @@ In determining the meaning of the direct base class specification `A` of a clas
 
 > *Example*: The following
 >
+> <!-- Example: {template:"standalone-lib", name:"RecursiveBaseClassSpecification", expectedErrors:["CS0146"]} -->
 > ```csharp
 > class X<T>
 > {
@@ -252,6 +258,7 @@ The base classes of a class are the direct base class and its base classes. In o
 
 > *Example*: In the following:
 >
+> <!-- Example: {template:"standalone-lib", name:"DirectBaseClasses", replaceEllipsis:true} -->
 > ```csharp
 > class A {...}
 > class B<T> : A {...}
@@ -269,12 +276,14 @@ It is a compile-time error for a class to depend on itself. For the purpose of t
 
 > *Example*: The example
 >
+> <!-- Example: {template:"standalone-lib", name:"SelfBaseClass", expectedErrors:["CS0146"]} -->
 > ```csharp
-> class A: A {}
+> class A : A {}
 > ```
 >
 > is erroneous because the class depends on itself. Likewise, the example
 >
+> <!-- Example: {template:"standalone-lib", name:"CircularBaseClass1", expectedErrors:["CS0146","CS0146","CS0146"]} -->
 > ```csharp
 > class A : B {}
 > class B : C {}
@@ -283,6 +292,7 @@ It is a compile-time error for a class to depend on itself. For the purpose of t
 >
 > is in error because the classes circularly depend on themselves. Finally, the example
 >
+> <!-- Example: {template:"standalone-lib", name:"CircularBaseClass2", expectedErrors:["CS0146","CS0146"]} -->
 > ```csharp
 > class A : B.C {}
 > class B : A
@@ -299,6 +309,7 @@ A class does not depend on the classes that are nested within it.
 
 > *Example*: In the following code
 >
+> <!-- Example: {template:"standalone-lib", name:"NestedClassDependency"} -->
 > ```csharp
 > class A
 > {
@@ -313,6 +324,7 @@ A class does not depend on the classes that are nested within it.
 It is not possible to derive from a sealed class.
 > *Example*: In the following code
 >
+> <!-- Example: {template:"standalone-lib", name:"DeriveFromSealedClass", expectedErrors:["CS0509"]} -->
 > ```csharp
 > sealed class A {}
 > class B : A {} // Error, cannot derive from a sealed class
@@ -843,6 +855,7 @@ All members of a generic class can use type parameters from any enclosing class,
 
 > *Example*:
 >
+> <!-- Example: {template:"standalone-console",name:"TypeParameterSubstitution",expectedOutput:["1","3.1415"]} -->
 > ```csharp
 > class C<V>
 > {
@@ -1277,6 +1290,8 @@ Both signatures are reserved, even if the property is read-only or write-only.
 
 > *Example*: In the following code
 >
+> <!-- TODO: Check why CS0109 (The member 'B.get_P()' does not hide an accessible member. The new keyword is not required.) is emitted. -->
+> <!-- Example: {template:"standalone-console",name:"PropertyReservedSignatures",expectedOutput:["123","123","456"],expectedWarnings:["CS0109","CS0109"]} -->
 > ```csharp
 > using System;
 > class A

tools/ExampleExtractor/Example.cs

@@ -0,0 +1,133 @@
+using Newtonsoft.Json;
+using System.Net.Http.Json;
+
+namespace ExampleExtractor;
+
+internal class Example
+{
+    private const string ExampleCommentPrefix = "<!-- Example: ";
+    private const string CommentSuffix = " -->";
+
+    internal ExampleMetadata Metadata { get; }
+
+    /// <summary>
+    /// The name of the example. This should be unique across all files.
+    /// </summary>
+    internal string Name => Metadata.Name;
+
+    /// <summary>
+    /// The name of the template to apply.
+    /// </summary>
+    internal string Template => Metadata.Template;
+
+    /// <summary>
+    /// The source location of the example.
+    /// </summary>
+    internal string Source => Metadata.Source;
+
+    /// <summary>
+    /// The code within the example.
+    /// </summary>
+    internal string Code { get; }
+
+    /// <summary>
+    /// Loads examples from all the Markdown files in the given directory.
+    /// </summary>
+    internal static List<Example> LoadExamplesFromDirectory(string directory) =>
+        Directory.GetFiles(directory, "*.md")
+            .SelectMany(LoadExamplesFromFile)
+            .ToList();
+
+    private Example(ExampleMetadata metadata, string code)
+    {
+        Metadata = metadata;
+        if (metadata.ReplaceEllipsis)
+        {
+            code = code.Replace("...", "/* ... */");
+        }
+        Code = code;
+    }
+
+    /// <summary>
+    /// Loads examples from a single Markdown file.
+    /// </summary>
+    private static IEnumerable<Example> LoadExamplesFromFile(string markdownFile)
+    {
+        string[] lines = File.ReadAllLines(markdownFile);
+
+        for (int i = 0; i < lines.Length; i++)
+        {
+            string line = lines[i];
+            if (!line.Contains(ExampleCommentPrefix))
+            {
+                continue;
+            }
+            var metadata = ParseComment(line);
+
+            string prefix = line.Substring(0, line.IndexOf(ExampleCommentPrefix));
+            string trimmedPrefix = prefix.Trim();
+
+            // We don't currently assume the example comes immediately after the comment.
+            // This could allow for expected output in another comment, for example.
+            // If it turns out not to be useful, we could just check that lines[i+1] ends with ```csharp
+            int openingLine = FindLineEnding(i, "```csharp"); // 0-based, and pre-code
+            int closingLine = FindLineEnding(openingLine, "```"); // 0-based, and post-code
+
+            var codeLines = lines
+                .Skip(openingLine + 1)
+                .Take(closingLine - openingLine - 1)
+                .Select(TrimPrefix);
+
+            string code = string.Join("\n", codeLines);
+
+            // Augment the metadata
+            metadata.StartLine = openingLine + 1;
+            metadata.EndLine = closingLine;
+            metadata.MarkdownFile = Path.GetFileName(markdownFile);
+
+            yield return new Example(metadata, code);
+            i = closingLine;
+
+            string TrimPrefix(string codeLine) =>
+                codeLine.StartsWith(prefix) ? codeLine.Substring(prefix.Length)
+                        : codeLine.StartsWith(trimmedPrefix) ? codeLine.Substring(trimmedPrefix.Length)
+                        : throw new InvalidOperationException($"Example in {markdownFile} starting at line {openingLine} contains line without common prefix");
+
+        }
+
+        int FindLineEnding(int start, string suffix)
+        {
+            for (int i = start; i < lines.Length; i++)
+            {
+                if (lines[i].EndsWith(suffix))
+                {
+                    return i;
+                }
+            }
+            throw new InvalidOperationException($"File {markdownFile} has no line ending '{suffix}' starting at line {start + 1}");
+        }
+
+        ExampleMetadata ParseComment(string commentLine)
+        {
+            int prefixIndex = commentLine.IndexOf(ExampleCommentPrefix);
+            if (prefixIndex == -1)
+            {
+                throw new ArgumentException($"'{commentLine}' does not contain {ExampleCommentPrefix}");
+            }
+            if (!commentLine.EndsWith(CommentSuffix))
+            {
+                throw new ArgumentException($"'{commentLine}' does not end with {CommentSuffix}");
+            }
+            string json = commentLine[(prefixIndex + ExampleCommentPrefix.Length)..^CommentSuffix.Length];
+            try
+            {
+                return JsonConvert.DeserializeObject<ExampleMetadata>(json) ?? throw new ArgumentException("Invalid (null) configuration");
+            }
+            catch (JsonException e)
+            {
+                // TODO: Add the source information as well.
+                throw new Exception($"Error parsing metadata '{json}'", e);
+            }
+        }
+    }
+}

tools/ExampleExtractor/ExampleExtractor.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+  </ItemGroup>
+
+</Project>

tools/ExampleExtractor/ExampleMetadata.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+#nullable disable
+
+namespace ExampleExtractor;
+
+/// <summary>
+/// Metadata about an example, from the configuration in the Markdown comment
+/// </summary>
+public class ExampleMetadata
+{
+    public const string MetadataFile = "metadata.json";
+
+    // Information loaded from the comment
+    public string Template { get; set; }
+    public string Name { get; set; }
+    public bool ReplaceEllipsis { get; set; }
+    public List<string> ExpectedErrors { get; set; }
+    public List<string> ExpectedWarnings { get; set; }
+    public List<string> ExpectedOutput { get; set; }
+
+    // Information provided by the example extractor
+    public string MarkdownFile { get; set; }
+    public int StartLine { get; set; }
+    public int EndLine { get; set; }
+
+    [JsonIgnore]
+    public string Source => $"{MarkdownFile}:{StartLine}-{EndLine}";
+}

tools/ExampleExtractor/Program.cs

@@ -0,0 +1,69 @@
+using ExampleExtractor;
+
+if (args.Length != 3)
+{
+    Console.WriteLine("Arguments: <markdown-directory> <template-directory> <output-directory>");
+    return 1;
+}
+
+string markdownDirectory = args[0];
+string templateDirectory = args[1];
+string outputDirectory = args[2];
+
+if (!ValidateDirectory(markdownDirectory) ||
+    !ValidateDirectory(templateDirectory))
+{
+    return 1;
+}
+
+if (Directory.Exists(outputDirectory))
+{
+    if (Directory.GetFiles(outputDirectory).Any())
+    {
+        Console.WriteLine($"Error: {outputDirectory} exists and contains files.");
+    }
+    var oldSubdirectories = Directory.GetDirectories(outputDirectory);
+    if (oldSubdirectories.Any())
+    {
+        Console.WriteLine($"Deleting old output subdirectories ({oldSubdirectories.Length})");
+        foreach (var subdirectory in oldSubdirectories)
+        {
+            Directory.Delete(subdirectory, true);
+        }
+    }
+}
+else
+{
+    Directory.CreateDirectory(outputDirectory);
+}
+
+var templates = Template.LoadTemplates(templateDirectory);
+Console.WriteLine($"Loaded {templates.Count} templates");
+var examples = Example.LoadExamplesFromDirectory(markdownDirectory);
+Console.WriteLine($"Loaded {examples.Count} examples");
+
+bool anyErrors = false;
+foreach (var example in examples)
+{
+    Console.WriteLine($"Processing example {example.Name} from {example.Source}");
+    if (!templates.TryGetValue(example.Template, out var template))
+    {
+        Console.WriteLine($"ERROR: template '{example.Template}' not found");
+        anyErrors = true;
+        continue;
+    }
+    template.Apply(example, outputDirectory);
+}
+Console.WriteLine("Finished example extraction.");
+
+return anyErrors ? 1 : 0;
+
+bool ValidateDirectory(string directory)
+{
+    if (!Directory.Exists(directory))
+    {
+        Console.WriteLine($"Error: '{directory}' does not exist or is not a directory");
+        return false;
+    }
+    return true;
+}