feature/assembler global toc

docs-builder.sln

@@ -73,6 +73,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler.Tests", "tests\docs-assembler.Tests\src\docs-assembler.Tests\docs-assembler.Tests.csproj", "{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -122,6 +124,10 @@ Global
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -138,5 +144,6 @@ Global
 		{6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
 	EndGlobalSection
 EndGlobal

src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs

@@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic)
 			_errors.Add(diagnostic);
 		else if (diagnostic.Severity == Severity.Warning)
 			_warnings.Add(diagnostic);
-		else
+		else if (!NoHints)
 			_hints.Add(diagnostic);
 	}
 

src/Elastic.Markdown/BuildContext.cs

@@ -29,6 +29,8 @@ public record BuildContext
 
 	public bool Force { get; init; }
 
+	public bool SkipMetadata { get; init; }
+
 	// This property is used to determine if the site should be indexed by search engines
 	public bool AllowIndexing { get; init; }
 
@@ -39,15 +41,6 @@ public string? UrlPathPrefix
 		init => _urlPathPrefix = value;
 	}
 
-	private readonly string? _staticUrlPathPrefix;
-	public string? StaticUrlPathPrefix
-	{
-		get => !string.IsNullOrWhiteSpace(_staticUrlPathPrefix)
-			? $"/{_staticUrlPathPrefix.Trim('/')}"
-			: UrlPathPrefix;
-		init => _staticUrlPathPrefix = value;
-	}
-
 	public BuildContext(IFileSystem fileSystem)
 		: this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { }
 

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -45,7 +45,7 @@ public interface ICrossLinkResolver
 public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver
 {
 	private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty;
-	private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new PreviewEnvironmentUriResolver();
+	private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new IsolatedBuildEnvironmentUriResolver();
 
 	public async Task<FetchedCrossLinks> FetchLinks()
 	{

src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs

@@ -9,7 +9,7 @@ public interface IUriEnvironmentResolver
 	Uri Resolve(Uri crossLinkUri, string path);
 }
 
-public class PreviewEnvironmentUriResolver : IUriEnvironmentResolver
+public class IsolatedBuildEnvironmentUriResolver : IUriEnvironmentResolver
 {
 	private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev");
 

src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs

@@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output
 
 	public HashSet<string> OffendingFiles { get; } = [];
 
-	public HashSet<string> InUseSubstitutionKeys { get; } = [];
+	public ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; } = [];
 
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
+	public bool NoHints { get; init; }
+
 	public Task StartAsync(Cancel cancellationToken)
 	{
 		if (_started is not null)
@@ -117,6 +119,8 @@ void Drain()
 		{
 			while (Channel.Reader.TryRead(out var item))
 			{
+				if (item.Severity == Severity.Hint && NoHints)
+					continue;
 				IncrementSeverityCount(item);
 				HandleItem(item);
 				_ = OffendingFiles.Add(item.File);
@@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item)
 			_ = Interlocked.Increment(ref _errors);
 		else if (item.Severity == Severity.Warning)
 			_ = Interlocked.Increment(ref _warnings);
-		else if (item.Severity == Severity.Hint)
+		else if (item.Severity == Severity.Hint && !NoHints)
 			_ = Interlocked.Increment(ref _hints);
 	}
 
@@ -175,5 +179,5 @@ public async ValueTask DisposeAsync()
 	}
 
 	public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>
-		_ = InUseSubstitutionKeys.Add(key.ToString());
+		_ = InUseSubstitutionKeys.TryAdd(key.ToString(), true);
 }

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -98,6 +98,28 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string
 		context.Collector.Channel.Write(d);
 	}
 
+	public static void EmitError(this DiagnosticsCollector collector, IFileInfo file, string message, Exception? e = null)
+	{
+		var d = new Diagnostic
+		{
+			Severity = Severity.Error,
+			File = file.FullName,
+			Message = message + (e != null ? Environment.NewLine + e : string.Empty),
+		};
+		collector.Channel.Write(d);
+	}
+
+	public static void EmitWarning(this DiagnosticsCollector collector, IFileInfo file, string message)
+	{
+		var d = new Diagnostic
+		{
+			Severity = Severity.Warning,
+			File = file.FullName,
+			Message = message,
+		};
+		collector.Channel.Write(d);
+	}
+
 	public static void EmitError(this IBlockExtension block, string message, Exception? e = null)
 	{
 		if (block.SkipValidation)

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -20,8 +20,14 @@ public interface IConversionCollector
 	void Collect(MarkdownFile file, MarkdownDocument document, string html);
 }
 
+public interface IDocumentationFileOutputProvider
+{
+	IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath);
+}
+
 public class DocumentationGenerator
 {
+	private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
 	private readonly IDocumentationFileExporter _documentationFileExporter;
@@ -34,10 +40,12 @@ public class DocumentationGenerator
 	public DocumentationGenerator(
 		DocumentationSet docSet,
 		ILoggerFactory logger,
+		IDocumentationFileOutputProvider? documentationFileOutputProvider = null,
 		IDocumentationFileExporter? documentationExporter = null,
 		IConversionCollector? conversionCollector = null
 	)
 	{
+		_documentationFileOutputProvider = documentationFileOutputProvider;
 		_writeFileSystem = docSet.Build.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
 
@@ -74,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 	public async Task GenerateAll(Cancel ctx)
 	{
 		var generationState = GetPreviousGenerationState();
-		if (Context.Force || generationState == null)
+		if (!Context.SkipMetadata && (Context.Force || generationState == null))
 			DocumentationSet.ClearOutputDirectory();
 
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
@@ -91,17 +99,22 @@ public async Task GenerateAll(Cancel ctx)
 
 		await ExtractEmbeddedStaticResources(ctx);
 
-		_logger.LogInformation($"Completing diagnostics channel");
-		Context.Collector.Channel.TryComplete();
+		if (Context.SkipMetadata)
+			return;
 
 		_logger.LogInformation($"Generating documentation compilation state");
 		await GenerateDocumentationState(ctx);
 
 		_logger.LogInformation($"Generating links.json");
 		await GenerateLinkReference(ctx);
+	}
 
+	public async Task StopDiagnosticCollection(Cancel ctx)
+	{
 		_logger.LogInformation($"Completing diagnostics channel");
+		Context.Collector.Channel.TryComplete();
 
+		_logger.LogInformation($"Stopping diagnostics collector");
 		await Context.Collector.StopAsync(ctx);
 
 		_logger.LogInformation($"Completed diagnostics channel");
@@ -140,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 	private void HintUnusedSubstitutionKeys()
 	{
 		var definedKeys = new HashSet<string>(Context.Configuration.Substitutions.Keys.ToArray());
-		var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray();
+		var inUse = new HashSet<string>(Context.Collector.InUseSubstitutionKeys.Keys);
+		var keysNotInUse = definedKeys.Except(inUse).ToArray();
 		// If we have less than 20 unused keys emit them separately
 		// Otherwise emit one hint with all of them for brevity
 		if (keysNotInUse.Length >= 20)
@@ -170,6 +184,8 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 			var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", $"_static{Path.DirectorySeparatorChar}");
 
 			var outputFile = OutputFile(path);
+			if (outputFile is null)
+				continue;
 			await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx);
 			_logger.LogDebug("Copied static embedded resource {Path}", path);
 		}
@@ -186,14 +202,21 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		}
 
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
+		//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
 		var outputFile = OutputFile(file.RelativePath);
-		await _documentationFileExporter.ProcessFile(file, outputFile, token);
+		if (outputFile is not null)
+			await _documentationFileExporter.ProcessFile(file, outputFile, token);
 	}
 
-	private IFileInfo OutputFile(string relativePath)
+	private IFileInfo? OutputFile(string relativePath)
 	{
 		var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath));
-		return outputFile;
+		if (relativePath.StartsWith("_static"))
+			return outputFile;
+
+		return _documentationFileOutputProvider is not null
+			? _documentationFileOutputProvider.OutputFile(DocumentationSet, outputFile, relativePath)
+			: outputFile;
 	}
 
 	private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<string> offendingFiles,

src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs

@@ -74,7 +74,7 @@ public ConfigurationFile(BuildContext context)
 		var redirectFile = new RedirectFile(redirectFileInfo, _context);
 		Redirects = redirectFile.Redirects;
 
-		var reader = new YamlStreamReader(sourceFile, _context);
+		var reader = new YamlStreamReader(sourceFile, _context.Collector);
 		try
 		{
 			foreach (var entry in reader.Read())
@@ -120,7 +120,7 @@ public ConfigurationFile(BuildContext context)
 			}
 
 			//we read it twice to ensure we read 'toc' last
-			reader = new YamlStreamReader(sourceFile, _context);
+			reader = new YamlStreamReader(sourceFile, _context.Collector);
 			foreach (var entry in reader.Read())
 			{
 				switch (entry.Key)

src/Elastic.Markdown/IO/Configuration/RedirectFile.cs

@@ -22,7 +22,7 @@ public RedirectFile(IFileInfo source, BuildContext context)
 		if (!source.Exists)
 			return;
 
-		var reader = new YamlStreamReader(Source, Context);
+		var reader = new YamlStreamReader(Source, Context.Collector);
 		try
 		{
 			foreach (var entry in reader.Read())

src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs

@@ -239,7 +239,7 @@ public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyVa
 		if (!found)
 			return null;
 
-		var tocYamlReader = new YamlStreamReader(source, _context);
+		var tocYamlReader = new YamlStreamReader(source, _context.Collector);
 		foreach (var kv in tocYamlReader.Read())
 		{
 			switch (kv.Key)

src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs

@@ -16,10 +16,10 @@ public record YamlToplevelKey
 	public required KeyValuePair<YamlNode, YamlNode> Entry { get; init; }
 }
 
-public class YamlStreamReader(IFileInfo source, BuildContext context)
+public class YamlStreamReader(IFileInfo source, DiagnosticsCollector collector)
 {
-	public IFileInfo Source { get; init; } = source;
-	public BuildContext Context { get; init; } = context;
+	private IFileInfo Source { get; init; } = source;
+	private DiagnosticsCollector Collector { get; init; } = collector;
 
 	public IEnumerable<YamlToplevelKey> Read()
 	{
@@ -30,7 +30,7 @@ public IEnumerable<YamlToplevelKey> Read()
 
 		if (yaml.Documents.Count == 0)
 		{
-			Context.EmitWarning(Source, "empty redirect file");
+			Collector.EmitWarning(Source, "empty redirect file");
 			yield break;
 		}
 		// Examine the stream
@@ -171,7 +171,7 @@ public void EmitWarning(string message, YamlNode? node) =>
 		EmitWarning(message, node?.Start, node?.End, (node as YamlScalarNode)?.Value?.Length);
 
 	public void EmitError(string message, Exception e) =>
-		Context.Collector.EmitError(Source.FullName, message, e);
+		Collector.EmitError(Source.FullName, message, e);
 
 	private void EmitError(string message, Mark? start = null, Mark? end = null, int? length = null)
 	{
@@ -185,7 +185,7 @@ private void EmitError(string message, Mark? start = null, Mark? end = null, int
 			Column = start.HasValue ? (int)start.Value.Column : null,
 			Length = length
 		};
-		Context.Collector.Channel.Write(d);
+		Collector.Channel.Write(d);
 	}
 	public void EmitWarning(string message, Mark? start = null, Mark? end = null, int? length = null)
 	{
@@ -199,6 +199,6 @@ public void EmitWarning(string message, Mark? start = null, Mark? end = null, in
 			Column = start.HasValue ? (int)start.Value.Column : null,
 			Length = length
 		};
-		Context.Collector.Channel.Write(d);
+		Collector.Channel.Write(d);
 	}
 }

src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs

@@ -46,21 +46,25 @@ public static GitCheckoutInformation Create(IDirectoryInfo source, IFileSystem f
 		}
 
 		var fakeRef = Guid.NewGuid().ToString()[..16];
-		var gitConfig = Git(source, ".git/config");
+		var gitConfig = Git(source, Path.Combine(".git", "config"));
 		if (!gitConfig.Exists)
 		{
-			logger?.LogInformation("Git checkout information not available.");
-			return Unavailable;
+			gitConfig = Git(source, Path.Combine("..", ".git", "config"));
+			if (!gitConfig.Exists)
+			{
+				logger?.LogInformation("Git checkout information not available.");
+				return Unavailable;
+			}
 		}
 
-		var head = Read(source, ".git/HEAD") ?? fakeRef;
+		var head = Read(source, Path.Combine(".git", "HEAD")) ?? fakeRef;
 		var gitRef = head;
 		var branch = head.Replace("refs/heads/", string.Empty);
 		//not detached HEAD
 		if (head.StartsWith("ref:"))
 		{
 			head = head.Replace("ref: ", string.Empty);
-			gitRef = Read(source, ".git/" + head) ?? fakeRef;
+			gitRef = Read(source, Path.Combine(".git", head)) ?? fakeRef;
 			branch = branch.Replace("ref: ", string.Empty);
 		}
 		else