Skip to content

feature/assembler global toc #770

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Mar 19, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs-builder.sln
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler.Tests", "tests\docs-assembler.Tests\src\docs-assembler.Tests\docs-assembler.Tests.csproj", "{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -122,6 +124,10 @@ Global
{C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU
{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
Expand All @@ -138,5 +144,6 @@ Global
{6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
{9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
{C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
EndGlobalSection
EndGlobal
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic)
_errors.Add(diagnostic);
else if (diagnostic.Severity == Severity.Warning)
_warnings.Add(diagnostic);
else
else if (!NoHints)
_hints.Add(diagnostic);
}

Expand Down
11 changes: 2 additions & 9 deletions src/Elastic.Markdown/BuildContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public record BuildContext

public bool Force { get; init; }

public bool SkipMetadata { get; init; }

// This property is used to determine if the site should be indexed by search engines
public bool AllowIndexing { get; init; }

Expand All @@ -39,15 +41,6 @@ public string? UrlPathPrefix
init => _urlPathPrefix = value;
}

private readonly string? _staticUrlPathPrefix;
public string? StaticUrlPathPrefix
{
get => !string.IsNullOrWhiteSpace(_staticUrlPathPrefix)
? $"/{_staticUrlPathPrefix.Trim('/')}"
: UrlPathPrefix;
init => _staticUrlPathPrefix = value;
}

public BuildContext(IFileSystem fileSystem)
: this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { }

Expand Down
2 changes: 1 addition & 1 deletion src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public interface ICrossLinkResolver
public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver
{
private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty;
private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new PreviewEnvironmentUriResolver();
private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new IsolatedBuildEnvironmentUriResolver();

public async Task<FetchedCrossLinks> FetchLinks()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public interface IUriEnvironmentResolver
Uri Resolve(Uri crossLinkUri, string path);
}

public class PreviewEnvironmentUriResolver : IUriEnvironmentResolver
public class IsolatedBuildEnvironmentUriResolver : IUriEnvironmentResolver
{
private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev");

Expand Down
10 changes: 7 additions & 3 deletions src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output

public HashSet<string> OffendingFiles { get; } = [];

public HashSet<string> InUseSubstitutionKeys { get; } = [];
public ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; } = [];

public ConcurrentBag<string> CrossLinks { get; } = [];

public bool NoHints { get; init; }

public Task StartAsync(Cancel cancellationToken)
{
if (_started is not null)
Expand Down Expand Up @@ -117,6 +119,8 @@ void Drain()
{
while (Channel.Reader.TryRead(out var item))
{
if (item.Severity == Severity.Hint && NoHints)
continue;
IncrementSeverityCount(item);
HandleItem(item);
_ = OffendingFiles.Add(item.File);
Expand All @@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item)
_ = Interlocked.Increment(ref _errors);
else if (item.Severity == Severity.Warning)
_ = Interlocked.Increment(ref _warnings);
else if (item.Severity == Severity.Hint)
else if (item.Severity == Severity.Hint && !NoHints)
_ = Interlocked.Increment(ref _hints);
}

Expand Down Expand Up @@ -175,5 +179,5 @@ public async ValueTask DisposeAsync()
}

public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>
_ = InUseSubstitutionKeys.Add(key.ToString());
_ = InUseSubstitutionKeys.TryAdd(key.ToString(), true);
}
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,28 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string
context.Collector.Channel.Write(d);
}

public static void EmitError(this DiagnosticsCollector collector, IFileInfo file, string message, Exception? e = null)
{
var d = new Diagnostic
{
Severity = Severity.Error,
File = file.FullName,
Message = message + (e != null ? Environment.NewLine + e : string.Empty),
};
collector.Channel.Write(d);
}

public static void EmitWarning(this DiagnosticsCollector collector, IFileInfo file, string message)
{
var d = new Diagnostic
{
Severity = Severity.Warning,
File = file.FullName,
Message = message,
};
collector.Channel.Write(d);
}

public static void EmitError(this IBlockExtension block, string message, Exception? e = null)
{
if (block.SkipValidation)
Expand Down
37 changes: 30 additions & 7 deletions src/Elastic.Markdown/DocumentationGenerator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,14 @@ public interface IConversionCollector
void Collect(MarkdownFile file, MarkdownDocument document, string html);
}

public interface IDocumentationFileOutputProvider
{
IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath);
}

public class DocumentationGenerator
{
private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider;
private readonly ILogger _logger;
private readonly IFileSystem _writeFileSystem;
private readonly IDocumentationFileExporter _documentationFileExporter;
Expand All @@ -34,10 +40,12 @@ public class DocumentationGenerator
public DocumentationGenerator(
DocumentationSet docSet,
ILoggerFactory logger,
IDocumentationFileOutputProvider? documentationFileOutputProvider = null,
IDocumentationFileExporter? documentationExporter = null,
IConversionCollector? conversionCollector = null
)
{
_documentationFileOutputProvider = documentationFileOutputProvider;
_writeFileSystem = docSet.Build.WriteFileSystem;
_logger = logger.CreateLogger(nameof(DocumentationGenerator));

Expand Down Expand Up @@ -74,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx)
public async Task GenerateAll(Cancel ctx)
{
var generationState = GetPreviousGenerationState();
if (Context.Force || generationState == null)
if (!Context.SkipMetadata && (Context.Force || generationState == null))
DocumentationSet.ClearOutputDirectory();

if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
Expand All @@ -91,17 +99,22 @@ public async Task GenerateAll(Cancel ctx)

await ExtractEmbeddedStaticResources(ctx);

_logger.LogInformation($"Completing diagnostics channel");
Context.Collector.Channel.TryComplete();
if (Context.SkipMetadata)
return;

_logger.LogInformation($"Generating documentation compilation state");
await GenerateDocumentationState(ctx);

_logger.LogInformation($"Generating links.json");
await GenerateLinkReference(ctx);
}

public async Task StopDiagnosticCollection(Cancel ctx)
{
_logger.LogInformation($"Completing diagnostics channel");
Context.Collector.Channel.TryComplete();

_logger.LogInformation($"Stopping diagnostics collector");
await Context.Collector.StopAsync(ctx);

_logger.LogInformation($"Completed diagnostics channel");
Expand Down Expand Up @@ -140,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
private void HintUnusedSubstitutionKeys()
{
var definedKeys = new HashSet<string>(Context.Configuration.Substitutions.Keys.ToArray());
var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray();
var inUse = new HashSet<string>(Context.Collector.InUseSubstitutionKeys.Keys);
var keysNotInUse = definedKeys.Except(inUse).ToArray();
// If we have less than 20 unused keys emit them separately
// Otherwise emit one hint with all of them for brevity
if (keysNotInUse.Length >= 20)
Expand Down Expand Up @@ -170,6 +184,8 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", $"_static{Path.DirectorySeparatorChar}");

var outputFile = OutputFile(path);
if (outputFile is null)
continue;
await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx);
_logger.LogDebug("Copied static embedded resource {Path}", path);
}
Expand All @@ -186,14 +202,21 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
}

_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
var outputFile = OutputFile(file.RelativePath);
await _documentationFileExporter.ProcessFile(file, outputFile, token);
if (outputFile is not null)
await _documentationFileExporter.ProcessFile(file, outputFile, token);
}

private IFileInfo OutputFile(string relativePath)
private IFileInfo? OutputFile(string relativePath)
{
var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath));
return outputFile;
if (relativePath.StartsWith("_static"))
return outputFile;

return _documentationFileOutputProvider is not null
? _documentationFileOutputProvider.OutputFile(DocumentationSet, outputFile, relativePath)
: outputFile;
}

private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<string> offendingFiles,
Expand Down
4 changes: 2 additions & 2 deletions src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public ConfigurationFile(BuildContext context)
var redirectFile = new RedirectFile(redirectFileInfo, _context);
Redirects = redirectFile.Redirects;

var reader = new YamlStreamReader(sourceFile, _context);
var reader = new YamlStreamReader(sourceFile, _context.Collector);
try
{
foreach (var entry in reader.Read())
Expand Down Expand Up @@ -120,7 +120,7 @@ public ConfigurationFile(BuildContext context)
}

//we read it twice to ensure we read 'toc' last
reader = new YamlStreamReader(sourceFile, _context);
reader = new YamlStreamReader(sourceFile, _context.Collector);
foreach (var entry in reader.Read())
{
switch (entry.Key)
Expand Down
2 changes: 1 addition & 1 deletion src/Elastic.Markdown/IO/Configuration/RedirectFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public RedirectFile(IFileInfo source, BuildContext context)
if (!source.Exists)
return;

var reader = new YamlStreamReader(Source, Context);
var reader = new YamlStreamReader(Source, Context.Collector);
try
{
foreach (var entry in reader.Read())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyVa
if (!found)
return null;

var tocYamlReader = new YamlStreamReader(source, _context);
var tocYamlReader = new YamlStreamReader(source, _context.Collector);
foreach (var kv in tocYamlReader.Read())
{
switch (kv.Key)
Expand Down
14 changes: 7 additions & 7 deletions src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ public record YamlToplevelKey
public required KeyValuePair<YamlNode, YamlNode> Entry { get; init; }
}

public class YamlStreamReader(IFileInfo source, BuildContext context)
public class YamlStreamReader(IFileInfo source, DiagnosticsCollector collector)
{
public IFileInfo Source { get; init; } = source;
public BuildContext Context { get; init; } = context;
private IFileInfo Source { get; init; } = source;
private DiagnosticsCollector Collector { get; init; } = collector;

public IEnumerable<YamlToplevelKey> Read()
{
Expand All @@ -30,7 +30,7 @@ public IEnumerable<YamlToplevelKey> Read()

if (yaml.Documents.Count == 0)
{
Context.EmitWarning(Source, "empty redirect file");
Collector.EmitWarning(Source, "empty redirect file");
yield break;
}
// Examine the stream
Expand Down Expand Up @@ -171,7 +171,7 @@ public void EmitWarning(string message, YamlNode? node) =>
EmitWarning(message, node?.Start, node?.End, (node as YamlScalarNode)?.Value?.Length);

public void EmitError(string message, Exception e) =>
Context.Collector.EmitError(Source.FullName, message, e);
Collector.EmitError(Source.FullName, message, e);

private void EmitError(string message, Mark? start = null, Mark? end = null, int? length = null)
{
Expand All @@ -185,7 +185,7 @@ private void EmitError(string message, Mark? start = null, Mark? end = null, int
Column = start.HasValue ? (int)start.Value.Column : null,
Length = length
};
Context.Collector.Channel.Write(d);
Collector.Channel.Write(d);
}
public void EmitWarning(string message, Mark? start = null, Mark? end = null, int? length = null)
{
Expand All @@ -199,6 +199,6 @@ public void EmitWarning(string message, Mark? start = null, Mark? end = null, in
Column = start.HasValue ? (int)start.Value.Column : null,
Length = length
};
Context.Collector.Channel.Write(d);
Collector.Channel.Write(d);
}
}
14 changes: 9 additions & 5 deletions src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,25 @@ public static GitCheckoutInformation Create(IDirectoryInfo source, IFileSystem f
}

var fakeRef = Guid.NewGuid().ToString()[..16];
var gitConfig = Git(source, ".git/config");
var gitConfig = Git(source, Path.Combine(".git", "config"));
if (!gitConfig.Exists)
{
logger?.LogInformation("Git checkout information not available.");
return Unavailable;
gitConfig = Git(source, Path.Combine("..", ".git", "config"));
if (!gitConfig.Exists)
{
logger?.LogInformation("Git checkout information not available.");
return Unavailable;
}
}

var head = Read(source, ".git/HEAD") ?? fakeRef;
var head = Read(source, Path.Combine(".git", "HEAD")) ?? fakeRef;
var gitRef = head;
var branch = head.Replace("refs/heads/", string.Empty);
//not detached HEAD
if (head.StartsWith("ref:"))
{
head = head.Replace("ref: ", string.Empty);
gitRef = Read(source, ".git/" + head) ?? fakeRef;
gitRef = Read(source, Path.Combine(".git", head)) ?? fakeRef;
branch = branch.Replace("ref: ", string.Empty);
}
else
Expand Down
Loading
Loading