Add multi-file functionality (for a single project) (#735)

This fixes almost all examples where multiple files are required, example for VersioningOfConstantsAndStaticReadonlyFields1/2, which actually require multiple *projects*.

A few examples marked as requiring multiple files actually only needed additional files. It would be nice to be able to grab code from other examples to avoid maintenance notes, but that can be done later.

Note that the additional files generated have no template, and therefore no implicit using directives - hence the addition of "using System.Diagnostics;" in a couple of examples. This is *slightly* unfortunate, but not a significant issue - and the alternative of specifying template files would be much more work.

With this change in place, we will have 470 tested examples.

Author: jskeet

standard/attributes.md

@@ -611,9 +611,10 @@ It is important to understand that the inclusion or exclusion of a call to a con
 
 > *Example*: In the following code
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib", name:"ConditionalMethods3", expectedOutput:["Executed Class1.F"]} -->
+> <!-- Example: {template:"standalone-lib", name:"ConditionalMethods3"} -->
 > ```csharp
-> // File class1.cs:
+> // File Class1.cs:
+> using System.Diagnostics;
 > class Class1
 > {
 >     [Conditional("DEBUG")]
@@ -623,7 +624,7 @@ It is important to understand that the inclusion or exclusion of a call to a con
 >     }
 > }
 > 
-> // File class2.cs:
+> // File Class2.cs:
 > #define DEBUG
 > class Class2
 > {
@@ -633,7 +634,7 @@ It is important to understand that the inclusion or exclusion of a call to a con
 >     }
 > }
 > 
-> // File class3.cs:
+> // File Class3.cs:
 > #undef DEBUG
 > class Class3
 > {
@@ -652,16 +653,17 @@ The use of conditional methods in an inheritance chain can be confusing. Calls m
 
 > *Example*: In the following code
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib", name:"ConditionalMethods4", expectedOutput:["Class2.M executed"]} -->
+> <!-- Example: {template:"standalone-console", name:"ConditionalMethods4", expectedOutput:["Class2.M executed"]} -->
 > ```csharp
-> // File class1.cs
+> // File Class1.cs
+> using System.Diagnostics;
 > class Class1
 > {
 >     [Conditional("DEBUG")]
 >     public virtual void M() => Console.WriteLine("Class1.M executed");
 > }
 > 
-> // File class2.cs
+> // File Class2.cs
 > class Class2 : Class1
 > {
 >     public override void M()
@@ -671,11 +673,11 @@ The use of conditional methods in an inheritance chain can be confusing. Calls m
 >     }
 > }
 > 
-> // File class3.cs
+> // File Class3.cs
 > #define DEBUG
 > class Class3
 > {
->     public static void Test()
+>     public static void Main()
 >     {
 >         Class2 c = new Class2();
 >         c.M(); // M is called
@@ -710,18 +712,19 @@ It is important to note that the inclusion or exclusion of an attribute specific
 
 > *Example*: In the example
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib", name:"ConditionalAttributeClasses2"} -->
+> <!-- Example: {template:"standalone-lib", name:"ConditionalAttributeClasses2"} -->
 > ```csharp
-> // File test.cs:
+> // File Test.cs:
+> using System.Diagnostics;
 > [Conditional("DEBUG")]
 > public class TestAttribute : Attribute {}
 > 
-> // File class1.cs:
+> // File Class1.cs:
 > #define DEBUG
 > [Test] // TestAttribute is specified
 > class Class1 {}
 > 
-> // File class2.cs:
+> // File Class2.cs:
 > #undef DEBUG
 > [Test] // TestAttribute is not specified
 > class Class2 {}

standard/classes.md

@@ -679,7 +679,7 @@ Nested types can be declared in multiple parts by using the `partial` modifier.
 
 > *Example*: The following partial class is implemented in two parts, which reside in different compilation units. The first part is machine generated by a database-mapping tool while the second part is manually authored:
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib-without-using", name:"PartialDeclarations1", replaceEllipsis:true, expectedWarnings:["CS0169","CS0169","CS0169","CS0649"], additionalFiles:["Order.cs"]} -->
+> <!-- Example: {template:"standalone-lib-without-using", name:"PartialDeclarations1", replaceEllipsis:true, expectedWarnings:["CS0169","CS0169","CS0169","CS0649"], additionalFiles:["Order.cs"]} -->
 > ```csharp
 > public partial class Customer
 > {
@@ -694,6 +694,7 @@ Nested types can be declared in multiple parts by using the `partial` modifier.
 >     }
 > }
 >
+> // File: Customer2.cs
 > public partial class Customer
 > {
 >     public void SubmitOrder(Order orderSubmitted) => orders.Add(orderSubmitted);
@@ -1596,7 +1597,7 @@ Constants and readonly fields have different binary versioning semantics. When a
 
 > *Example*: Consider an application that consists of two separate programs:
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib-without-using", name:"VersioningOfConstantsAndStaticReadonlyFields1"} -->
+> <!-- RequiresSeparateProjects$Example: {template:"standalone-lib-without-using", name:"VersioningOfConstantsAndStaticReadonlyFields1"} -->
 > ```csharp
 > namespace Program1
 > {
@@ -1609,7 +1610,7 @@ Constants and readonly fields have different binary versioning semantics. When a
 >
 > and
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-console", name:"VersioningOfConstantsAndStaticReadonlyFields2", expectedOutput:["x", "x", "x"], expectedErrors:["x","x"], expectedWarnings:["x","x"]} -->
+> <!-- RequiresSeparateProjects$Example: {template:"standalone-console", name:"VersioningOfConstantsAndStaticReadonlyFields2", expectedOutput:["x", "x", "x"], expectedErrors:["x","x"], expectedWarnings:["x","x"]} -->
 > ```csharp
 > namespace Program2
 > {
@@ -2793,20 +2794,17 @@ An implementing partial method declaration can appear in the same part as the co
 
 Only a defining partial method participates in overload resolution. Thus, whether or not an implementing declaration is given, invocation expressions may resolve to invocations of the partial method. Because a partial method always returns `void`, such invocation expressions will always be expression statements. Furthermore, because a partial method is implicitly `private`, such statements will always occur within one of the parts of the type declaration within which the partial method is declared.
 
-> *Note*: The definition of matching defining and implementing partial method declarations does not require parameter names to match. This can produce *surprising*, albeit *well defined*, behaviour when named arguments ([§11.6.2.1](expressions.md#11621-general)) are used. For example, given the defining partial method declaration for `M`:
+> *Note*: The definition of matching defining and implementing partial method declarations does not require parameter names to match. This can produce *surprising*, albeit *well defined*, behaviour when named arguments ([§11.6.2.1](expressions.md#11621-general)) are used. For example, given the defining partial method declaration for `M` in one file, and the implementing partial method declaration in another file:
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib-without-using", name:"PartialMethods1"} -->
+> <!-- Example: {template:"standalone-lib-without-using", name:"PartialMethods1", "expectedErrors":["CS1739"], "expectedWarnings":["CS8826"]} -->
 > ```csharp
+> // File P1.cs:
 > partial class P
 > {
 >     static partial void M(int x);
 > }
-> ```
->
-> Then the implementing partial method declaration and invocation in other file:
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-lib-without-using", name:"PartialMethods2", expectedErrors:["x","x"], expectedWarnings:["x","x"]} -->
-> ```csharp
+> // File P2.cs:
 > partial class P
 > {
 >     static void Caller() => M(y: 0);
@@ -2841,6 +2839,7 @@ If a defining declaration but not an implementing declaration is given for a par
 Partial methods are useful for allowing one part of a type declaration to customize the behavior of another part, e.g., one that is generated by a tool. Consider the following partial class declaration:
 
 <!-- Example: {template:"standalone-lib-without-using", name:"PartialMethods3"} -->
+<!-- Maintenance Note: This code exists in additional-files as "Customer.cs" to be used in an example below. As such, certain changes to this example should be reflected in that file, in which case, *all* examples using that file should be tested. -->
 ```csharp
 partial class Customer
 {
@@ -2880,7 +2879,7 @@ class Customer
 
 Assume that another part is given, however, which provides implementing declarations of the partial methods:
 
-<!-- RequiresSeparateFiles$Example: {template:"standalone-lib", name:"PartialMethods5", expectedErrors:["x","x"], expectedWarnings:["x","x"]} -->
+<!-- Example: {template:"standalone-lib", name:"PartialMethods5", additionalFiles:["Customer.cs"]} -->
 ```csharp
 partial class Customer
 {
@@ -2926,6 +2925,7 @@ When the first parameter of a method includes the `this` modifier, that method i
 > *Example*: The following is an example of a static class that declares two extension methods:
 >
 > <!-- Example: {template:"standalone-lib", name:"ExtensionMethods1"} -->
+> <!-- Maintenance Note: This code exists in additional-files as "Extensions.cs" to be used in the examples below. As such, certain changes to this example should be reflected in that file, in which case, *all* examples using that file should be tested. -->
 > ```csharp
 > public static class Extensions
 > {
@@ -2950,7 +2950,7 @@ An extension method is a regular static method. In addition, where its enclosing
 
 > *Example*: The following program uses the extension methods declared above:
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-console", name:"ExtensionMethods2", expectedOutput:["x", "x", "x"], expectedErrors:["x","x"], expectedWarnings:["x","x"]} -->
+> <!-- Example: {template:"standalone-console", name:"ExtensionMethods2", additionalFiles:["Extensions.cs"], expectedOutput:["22", "333"]} -->
 > ```csharp
 > static class Program
 > {
@@ -2967,7 +2967,7 @@ An extension method is a regular static method. In addition, where its enclosing
 >
 > The `Slice` method is available on the `string[]`, and the `ToInt32` method is available on `string`, because they have been declared as extension methods. The meaning of the program is the same as the following, using ordinary static method calls:
 >
-> <!-- RequiresSeparateFiles$Example: {template:"standalone-console", name:"ExtensionMethods3", expectedOutput:["x", "x", "x"], expectedErrors:["x","x"], expectedWarnings:["x","x"]} -->
+> <!-- Example: {template:"standalone-console", name:"ExtensionMethods3", additionalFiles:["Extensions.cs"], expectedOutput:["22", "333"]} -->
 > ```csharp
 > static class Program
 > {

standard/namespaces.md

@@ -32,9 +32,9 @@ The *namespace_member_declaration*s of each compilation unit of a program contri
 > <!-- Example: {template:"standalone-lib-without-using", name:"CompilationUnits"} -->
 > ```csharp
 > // File A.cs:
->     class A {}
+> class A {}
 > // File B.cs:
->     class B {}
+> class B {}
 > ```
 >
 > The two compilation units contribute to the single global namespace, in this case declaring two classes with the fully qualified names `A` and `B`. Because the two compilation units contribute to the same declaration space, it would have been an error if each contained a declaration of a member with the same name.

tools/ExampleExtractor/Template.cs

@@ -7,6 +7,7 @@ internal class Template
     private const string AdditionalFilesDirectory = "additional-files";
     private const string ExampleCodeSubstitution = "$example-code";
     private const string ExampleNameSubstitution = "$example-name";
+    private const string FileCommentPrefix = "// File ";
 
     internal string Name { get; }
     private readonly Dictionary<string, string> files;
@@ -27,13 +28,16 @@ internal void Apply(Example example, string rootOutputDirectory, string rootTemp
     {
         var outputDirectory = Path.Combine(rootOutputDirectory, example.Name);
         Directory.CreateDirectory(outputDirectory);
+
+        var code = ExtractExtraFiles(example.Code, outputDirectory);
+
         foreach (var pair in files)
         {
             string file = Path.Combine(outputDirectory, pair.Key);
-            string code = pair.Value
-                .Replace(ExampleCodeSubstitution, example.Code)
+            string content = pair.Value
+                .Replace(ExampleCodeSubstitution, code)
                 .Replace(ExampleNameSubstitution, example.Name);
-            File.WriteAllText(file, code);
+            File.WriteAllText(file, content);
         }
         if (example.Metadata.AdditionalFiles is List<string> additionalFiles)
         {
@@ -48,6 +52,43 @@ internal void Apply(Example example, string rootOutputDirectory, string rootTemp
         File.WriteAllText(Path.Combine(outputDirectory, ExampleMetadata.MetadataFile), metadataJson);
     }
 
+    /// <summary>
+    /// Returns all the code before the first "// File:" comment, extracting any additional
+    /// files into the given directory.
+    /// </summary>
+    private string ExtractExtraFiles(string code, string outputDirectory)
+    {
+        var lines = code.Split('\n').ToList();
+        // The implementation is a lot simpler if we know we've got an end marker.
+        lines.Add(FileCommentPrefix + "IgnoreMe");
+        string? currentFile = null;
+        List<string> currentLines = new List<string>();
+        string? initialCode = null;
+
+        foreach (var line in lines)
+        {
+            if (line.StartsWith(FileCommentPrefix))
+            {
+                if (currentFile is null)
+                {
+                    // Remember this for later.
+                    initialCode = string.Join("\n", currentLines);
+                }
+                else
+                {
+                    File.WriteAllLines(currentFile, currentLines);
+                }
+                currentLines.Clear();
+                currentFile = Path.Combine(outputDirectory, line[FileCommentPrefix.Length..].TrimEnd(':'));
+            }
+            else
+            {
+                currentLines.Add(line);
+            }
+        }
+        return initialCode ?? throw new InvalidOperationException($"Never saw a file terminator - bug in {nameof(ExtractExtraFiles)}");
+    }
+
     internal static Dictionary<string, Template> LoadTemplates(string directory) =>
         Directory.GetDirectories(directory)
             .Select(LoadTemplate)

tools/ExampleTester/GeneratedExample.cs

@@ -35,10 +35,11 @@ private static GeneratedExample Load(string directory)
 
     internal async Task<bool> Test(TesterConfiguration configuration)
     {
-        var outputLines = new List<string>();
-        outputLines.Add($"Testing {Metadata.Name} from {Metadata.Source}");
+        var outputLines = new List<string> { $"Testing {Metadata.Name} from {Metadata.Source}" };
 
-        using var workspace = MSBuildWorkspace.Create();
+        // Explicitly do a release build, to avoid implicitly defining DEBUG.
+        var properties = new Dictionary<string, string> { { "Configuration", "Release" } };
+        using var workspace = MSBuildWorkspace.Create(properties);
         // TODO: Validate this more cleanly.
         var projectFile = Metadata.Project is string specifiedProject
             ? Path.Combine(directory, $"{specifiedProject}.csproj")

tools/example-templates/additional-files/Customer.cs

@@ -0,0 +1,18 @@
+partial class Customer
+{
+    string name;
+
+    public string Name
+    {
+        get => name;
+        set
+        {
+            OnNameChanging(value);
+            name = value;
+            OnNameChanged();
+        }
+    }
+
+    partial void OnNameChanging(string newName);
+    partial void OnNameChanged();
+}

tools/example-templates/additional-files/Extensions.cs

@@ -0,0 +1,15 @@
+public static class Extensions
+{
+    public static int ToInt32(this string s) => Int32.Parse(s);
+
+    public static T[] Slice<T>(this T[] source, int index, int count)
+    {
+        if (index < 0 || count < 0 || source.Length - index < count)
+        {
+            throw new ArgumentException();
+        }
+        T[] result = new T[count];
+        Array.Copy(source, index, result, 0, count);
+        return result;
+    }
+}