diff --git a/docs-builder.sln b/docs-builder.sln index d8d67fc4c..f300cc7c0 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -73,6 +73,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler.Tests", "tests\docs-assembler.Tests\src\docs-assembler.Tests\docs-assembler.Tests.csproj", "{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -122,6 +124,10 @@ Global {C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU {C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU {C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} @@ -138,5 +144,6 @@ Global {6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5} EndGlobalSection EndGlobal diff --git a/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs b/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs index d40e285b8..5aa5ce497 100644 --- a/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs +++ b/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs @@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic) _errors.Add(diagnostic); else if (diagnostic.Severity == Severity.Warning) _warnings.Add(diagnostic); - else + else if (!NoHints) _hints.Add(diagnostic); } diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs index bd232b8c2..8eb18e002 100644 --- a/src/Elastic.Markdown/BuildContext.cs +++ b/src/Elastic.Markdown/BuildContext.cs @@ -29,6 +29,8 @@ public record BuildContext public bool Force { get; init; } + public bool SkipMetadata { get; init; } + // This property is used to determine if the site should be indexed by search engines public bool AllowIndexing { get; init; } @@ -39,15 +41,6 @@ public string? UrlPathPrefix init => _urlPathPrefix = value; } - private readonly string? _staticUrlPathPrefix; - public string? StaticUrlPathPrefix - { - get => !string.IsNullOrWhiteSpace(_staticUrlPathPrefix) - ? $"/{_staticUrlPathPrefix.Trim('/')}" - : UrlPathPrefix; - init => _staticUrlPathPrefix = value; - } - public BuildContext(IFileSystem fileSystem) : this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { } diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs index 701385d90..8eb158704 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs @@ -45,7 +45,7 @@ public interface ICrossLinkResolver public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver { private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; - private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new PreviewEnvironmentUriResolver(); + private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new IsolatedBuildEnvironmentUriResolver(); public async Task FetchLinks() { diff --git a/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs b/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs index 4bad70808..d95e0deaa 100644 --- a/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs @@ -9,7 +9,7 @@ public interface IUriEnvironmentResolver Uri Resolve(Uri crossLinkUri, string path); } -public class PreviewEnvironmentUriResolver : IUriEnvironmentResolver +public class IsolatedBuildEnvironmentUriResolver : IUriEnvironmentResolver { private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev"); diff --git a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs index 7a67e643c..c04101d1f 100644 --- a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs +++ b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs @@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection output public HashSet OffendingFiles { get; } = []; - public HashSet InUseSubstitutionKeys { get; } = []; + public ConcurrentDictionary InUseSubstitutionKeys { get; } = []; public ConcurrentBag CrossLinks { get; } = []; + public bool NoHints { get; init; } + public Task StartAsync(Cancel cancellationToken) { if (_started is not null) @@ -117,6 +119,8 @@ void Drain() { while (Channel.Reader.TryRead(out var item)) { + if (item.Severity == Severity.Hint && NoHints) + continue; IncrementSeverityCount(item); HandleItem(item); _ = OffendingFiles.Add(item.File); @@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item) _ = Interlocked.Increment(ref _errors); else if (item.Severity == Severity.Warning) _ = Interlocked.Increment(ref _warnings); - else if (item.Severity == Severity.Hint) + else if (item.Severity == Severity.Hint && !NoHints) _ = Interlocked.Increment(ref _hints); } @@ -175,5 +179,5 @@ public async ValueTask DisposeAsync() } public void CollectUsedSubstitutionKey(ReadOnlySpan key) => - _ = InUseSubstitutionKeys.Add(key.ToString()); + _ = InUseSubstitutionKeys.TryAdd(key.ToString(), true); } diff --git a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs index 6c0df2698..d2f5e3314 100644 --- a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs +++ b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs @@ -98,6 +98,28 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string context.Collector.Channel.Write(d); } + public static void EmitError(this DiagnosticsCollector collector, IFileInfo file, string message, Exception? e = null) + { + var d = new Diagnostic + { + Severity = Severity.Error, + File = file.FullName, + Message = message + (e != null ? Environment.NewLine + e : string.Empty), + }; + collector.Channel.Write(d); + } + + public static void EmitWarning(this DiagnosticsCollector collector, IFileInfo file, string message) + { + var d = new Diagnostic + { + Severity = Severity.Warning, + File = file.FullName, + Message = message, + }; + collector.Channel.Write(d); + } + public static void EmitError(this IBlockExtension block, string message, Exception? e = null) { if (block.SkipValidation) diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index b818b03ab..eb1756ebc 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -20,8 +20,14 @@ public interface IConversionCollector void Collect(MarkdownFile file, MarkdownDocument document, string html); } +public interface IDocumentationFileOutputProvider +{ + IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath); +} + public class DocumentationGenerator { + private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider; private readonly ILogger _logger; private readonly IFileSystem _writeFileSystem; private readonly IDocumentationFileExporter _documentationFileExporter; @@ -34,10 +40,12 @@ public class DocumentationGenerator public DocumentationGenerator( DocumentationSet docSet, ILoggerFactory logger, + IDocumentationFileOutputProvider? documentationFileOutputProvider = null, IDocumentationFileExporter? documentationExporter = null, IConversionCollector? conversionCollector = null ) { + _documentationFileOutputProvider = documentationFileOutputProvider; _writeFileSystem = docSet.Build.WriteFileSystem; _logger = logger.CreateLogger(nameof(DocumentationGenerator)); @@ -74,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx) public async Task GenerateAll(Cancel ctx) { var generationState = GetPreviousGenerationState(); - if (Context.Force || generationState == null) + if (!Context.SkipMetadata && (Context.Force || generationState == null)) DocumentationSet.ClearOutputDirectory(); if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges)) @@ -91,17 +99,22 @@ public async Task GenerateAll(Cancel ctx) await ExtractEmbeddedStaticResources(ctx); - _logger.LogInformation($"Completing diagnostics channel"); - Context.Collector.Channel.TryComplete(); + if (Context.SkipMetadata) + return; _logger.LogInformation($"Generating documentation compilation state"); await GenerateDocumentationState(ctx); _logger.LogInformation($"Generating links.json"); await GenerateLinkReference(ctx); + } + public async Task StopDiagnosticCollection(Cancel ctx) + { _logger.LogInformation($"Completing diagnostics channel"); + Context.Collector.Channel.TryComplete(); + _logger.LogInformation($"Stopping diagnostics collector"); await Context.Collector.StopAsync(ctx); _logger.LogInformation($"Completed diagnostics channel"); @@ -140,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => private void HintUnusedSubstitutionKeys() { var definedKeys = new HashSet(Context.Configuration.Substitutions.Keys.ToArray()); - var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray(); + var inUse = new HashSet(Context.Collector.InUseSubstitutionKeys.Keys); + var keysNotInUse = definedKeys.Except(inUse).ToArray(); // If we have less than 20 unused keys emit them separately // Otherwise emit one hint with all of them for brevity if (keysNotInUse.Length >= 20) @@ -170,6 +184,8 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx) var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", $"_static{Path.DirectorySeparatorChar}"); var outputFile = OutputFile(path); + if (outputFile is null) + continue; await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx); _logger.LogDebug("Copied static embedded resource {Path}", path); } @@ -186,14 +202,21 @@ private async Task ProcessFile(HashSet offendingFiles, DocumentationFile } _logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName); + //TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml var outputFile = OutputFile(file.RelativePath); - await _documentationFileExporter.ProcessFile(file, outputFile, token); + if (outputFile is not null) + await _documentationFileExporter.ProcessFile(file, outputFile, token); } - private IFileInfo OutputFile(string relativePath) + private IFileInfo? OutputFile(string relativePath) { var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath)); - return outputFile; + if (relativePath.StartsWith("_static")) + return outputFile; + + return _documentationFileOutputProvider is not null + ? _documentationFileOutputProvider.OutputFile(DocumentationSet, outputFile, relativePath) + : outputFile; } private bool CompilationNotNeeded(GenerationState? generationState, out HashSet offendingFiles, diff --git a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs index 5522bc3eb..b0e0d8aec 100644 --- a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs +++ b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs @@ -74,7 +74,7 @@ public ConfigurationFile(BuildContext context) var redirectFile = new RedirectFile(redirectFileInfo, _context); Redirects = redirectFile.Redirects; - var reader = new YamlStreamReader(sourceFile, _context); + var reader = new YamlStreamReader(sourceFile, _context.Collector); try { foreach (var entry in reader.Read()) @@ -120,7 +120,7 @@ public ConfigurationFile(BuildContext context) } //we read it twice to ensure we read 'toc' last - reader = new YamlStreamReader(sourceFile, _context); + reader = new YamlStreamReader(sourceFile, _context.Collector); foreach (var entry in reader.Read()) { switch (entry.Key) diff --git a/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs b/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs index 1b84024a1..9c24d2066 100644 --- a/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs +++ b/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs @@ -22,7 +22,7 @@ public RedirectFile(IFileInfo source, BuildContext context) if (!source.Exists) return; - var reader = new YamlStreamReader(Source, Context); + var reader = new YamlStreamReader(Source, Context.Collector); try { foreach (var entry in reader.Read()) diff --git a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs index 0737bbaaf..e5a8c6857 100644 --- a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs +++ b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs @@ -239,7 +239,7 @@ public IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyVa if (!found) return null; - var tocYamlReader = new YamlStreamReader(source, _context); + var tocYamlReader = new YamlStreamReader(source, _context.Collector); foreach (var kv in tocYamlReader.Read()) { switch (kv.Key) diff --git a/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs b/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs index 502bfbcdd..c85d5836e 100644 --- a/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs +++ b/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs @@ -16,10 +16,10 @@ public record YamlToplevelKey public required KeyValuePair Entry { get; init; } } -public class YamlStreamReader(IFileInfo source, BuildContext context) +public class YamlStreamReader(IFileInfo source, DiagnosticsCollector collector) { - public IFileInfo Source { get; init; } = source; - public BuildContext Context { get; init; } = context; + private IFileInfo Source { get; init; } = source; + private DiagnosticsCollector Collector { get; init; } = collector; public IEnumerable Read() { @@ -30,7 +30,7 @@ public IEnumerable Read() if (yaml.Documents.Count == 0) { - Context.EmitWarning(Source, "empty redirect file"); + Collector.EmitWarning(Source, "empty redirect file"); yield break; } // Examine the stream @@ -171,7 +171,7 @@ public void EmitWarning(string message, YamlNode? node) => EmitWarning(message, node?.Start, node?.End, (node as YamlScalarNode)?.Value?.Length); public void EmitError(string message, Exception e) => - Context.Collector.EmitError(Source.FullName, message, e); + Collector.EmitError(Source.FullName, message, e); private void EmitError(string message, Mark? start = null, Mark? end = null, int? length = null) { @@ -185,7 +185,7 @@ private void EmitError(string message, Mark? start = null, Mark? end = null, int Column = start.HasValue ? (int)start.Value.Column : null, Length = length }; - Context.Collector.Channel.Write(d); + Collector.Channel.Write(d); } public void EmitWarning(string message, Mark? start = null, Mark? end = null, int? length = null) { @@ -199,6 +199,6 @@ public void EmitWarning(string message, Mark? start = null, Mark? end = null, in Column = start.HasValue ? (int)start.Value.Column : null, Length = length }; - Context.Collector.Channel.Write(d); + Collector.Channel.Write(d); } } diff --git a/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs b/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs index 0d6996a57..45a23140a 100644 --- a/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs +++ b/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs @@ -46,21 +46,25 @@ public static GitCheckoutInformation Create(IDirectoryInfo source, IFileSystem f } var fakeRef = Guid.NewGuid().ToString()[..16]; - var gitConfig = Git(source, ".git/config"); + var gitConfig = Git(source, Path.Combine(".git", "config")); if (!gitConfig.Exists) { - logger?.LogInformation("Git checkout information not available."); - return Unavailable; + gitConfig = Git(source, Path.Combine("..", ".git", "config")); + if (!gitConfig.Exists) + { + logger?.LogInformation("Git checkout information not available."); + return Unavailable; + } } - var head = Read(source, ".git/HEAD") ?? fakeRef; + var head = Read(source, Path.Combine(".git", "HEAD")) ?? fakeRef; var gitRef = head; var branch = head.Replace("refs/heads/", string.Empty); //not detached HEAD if (head.StartsWith("ref:")) { head = head.Replace("ref: ", string.Empty); - gitRef = Read(source, ".git/" + head) ?? fakeRef; + gitRef = Read(source, Path.Combine(".git", head)) ?? fakeRef; branch = branch.Replace("ref: ", string.Empty); } else diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 156998c38..fd0af16ff 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -10,6 +10,7 @@ using Elastic.Markdown.Extensions; using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.IO.Configuration; +using Elastic.Markdown.IO.Discovery; using Elastic.Markdown.IO.Navigation; using Elastic.Markdown.Myst; using Microsoft.Extensions.Logging; @@ -81,7 +82,7 @@ public DocumentationSet(BuildContext build, ILoggerFactory logger, ICrossLinkRes }; MarkdownParser = new MarkdownParser(build, resolver); - Name = SourceDirectory.FullName; + Name = Build.Git.RepositoryName ?? "unavailable"; OutputStateFile = OutputDirectory.FileSystem.FileInfo.New(Path.Combine(OutputDirectory.FullName, ".doc.state")); LinkReferenceFile = OutputDirectory.FileSystem.FileInfo.New(Path.Combine(OutputDirectory.FullName, "links.json")); @@ -90,7 +91,7 @@ public DocumentationSet(BuildContext build, ILoggerFactory logger, ICrossLinkRes .SelectMany(extension => extension.ScanDocumentationFiles(ScanDocumentationFiles, DefaultFileHandling)) .ToArray(); - Files = files.Concat(additionalSources).ToArray(); + Files = files.Concat(additionalSources).Where(f => f is not ExcludedFile).ToArray(); LastWrite = Files.Max(f => f.SourceFile.LastWriteTimeUtc); @@ -153,7 +154,7 @@ private DocumentationFile DefaultFileHandling(IFileInfo file, IDirectoryInfo sou if (documentationFile is not null) return documentationFile; } - return new StaticFile(file, sourceDirectory); + return new ExcludedFile(file, sourceDirectory); } private void ValidateRedirectsExists() diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/Slices/HtmlWriter.cs index f77590d79..7fde86872 100644 --- a/src/Elastic.Markdown/Slices/HtmlWriter.cs +++ b/src/Elastic.Markdown/Slices/HtmlWriter.cs @@ -100,7 +100,6 @@ public async Task RenderLayout(MarkdownFile markdown, MarkdownDocument d TopLevelNavigationItems = [.. topLevelNavigationItems], NavigationHtml = navigationHtml, UrlPathPrefix = markdown.UrlPathPrefix, - StaticUrlPathPrefix = DocumentationSet.Build.StaticUrlPathPrefix, Applies = markdown.YamlFrontMatter?.AppliesTo, GithubEditUrl = editUrl, AllowIndexing = DocumentationSet.Build.AllowIndexing && !markdown.Hidden, diff --git a/src/Elastic.Markdown/Slices/Index.cshtml b/src/Elastic.Markdown/Slices/Index.cshtml index 07ef547f9..6a181cc02 100644 --- a/src/Elastic.Markdown/Slices/Index.cshtml +++ b/src/Elastic.Markdown/Slices/Index.cshtml @@ -13,7 +13,6 @@ NavigationHtml = Model.NavigationHtml, TopLevelNavigationItems = Model.TopLevelNavigationItems, UrlPathPrefix = Model.UrlPathPrefix, - StaticUrlPathPrefix = Model.StaticUrlPathPrefix, GithubEditUrl = Model.GithubEditUrl, AllowIndexing = Model.AllowIndexing, Features = Model.Features, diff --git a/src/Elastic.Markdown/Slices/_ViewModels.cs b/src/Elastic.Markdown/Slices/_ViewModels.cs index 57ded354c..68c40a0c9 100644 --- a/src/Elastic.Markdown/Slices/_ViewModels.cs +++ b/src/Elastic.Markdown/Slices/_ViewModels.cs @@ -23,7 +23,6 @@ public class IndexViewModel public required string NavigationHtml { get; init; } public required string? UrlPathPrefix { get; init; } - public required string? StaticUrlPathPrefix { get; init; } public required string? GithubEditUrl { get; init; } public required ApplicableTo? Applies { get; init; } public required bool AllowIndexing { get; init; } @@ -46,7 +45,6 @@ public class LayoutViewModel public required MarkdownFile? Previous { get; init; } public required MarkdownFile? Next { get; init; } public required string NavigationHtml { get; init; } - public required string? StaticUrlPathPrefix { get; init; } public required string? UrlPathPrefix { get; init; } public required string? GithubEditUrl { get; init; } public required bool AllowIndexing { get; init; } @@ -72,8 +70,8 @@ public string Static(string path) var staticPath = $"_static/{path.TrimStart('/')}"; var contentHash = StaticFileContentHashProvider.GetContentHash(path.TrimStart('/')); return string.IsNullOrEmpty(contentHash) - ? $"{StaticUrlPathPrefix}/{staticPath}" - : $"{StaticUrlPathPrefix}/{staticPath}?v={contentHash}"; + ? $"{UrlPathPrefix}/{staticPath}" + : $"{UrlPathPrefix}/{staticPath}?v={contentHash}"; } public required StaticFileContentHashProvider StaticFileContentHashProvider { get; init; } diff --git a/src/docs-assembler/AssembleContext.cs b/src/docs-assembler/AssembleContext.cs index a262bf5c2..70ab0765d 100644 --- a/src/docs-assembler/AssembleContext.cs +++ b/src/docs-assembler/AssembleContext.cs @@ -21,6 +21,8 @@ public class AssembleContext public IFileInfo ConfigurationPath { get; } + public IFileInfo NavigationPath { get; } + public IDirectoryInfo CheckoutDirectory { get; set; } public IDirectoryInfo OutputDirectory { get; set; } @@ -47,9 +49,14 @@ public AssembleContext( // This will live in docs-content soon if (!ReadFileSystem.File.Exists(configPath)) ExtractAssemblerConfiguration(configPath); - ConfigurationPath = ReadFileSystem.FileInfo.New(configPath); Configuration = AssemblyConfiguration.Deserialize(ReadFileSystem.File.ReadAllText(ConfigurationPath.FullName)); + + var navigationPath = Path.Combine(Paths.Root.FullName, "src", "docs-assembler", "navigation.yml"); + if (!ReadFileSystem.File.Exists(navigationPath)) + ExtractAssemblerConfiguration(navigationPath); + NavigationPath = ReadFileSystem.FileInfo.New(navigationPath); + CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? ".artifacts/checkouts"); OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? ".artifacts/assembly"); } diff --git a/src/docs-assembler/Building/AssemblerBuilder.cs b/src/docs-assembler/Building/AssemblerBuilder.cs index 367ea0e83..71eec9a75 100644 --- a/src/docs-assembler/Building/AssemblerBuilder.cs +++ b/src/docs-assembler/Building/AssemblerBuilder.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Documentation.Assembler.Configuration; +using Documentation.Assembler.Navigation; using Documentation.Assembler.Sourcing; using Elastic.Markdown; using Elastic.Markdown.CrossLinks; @@ -11,26 +12,33 @@ namespace Documentation.Assembler.Building; -public class AssemblerBuilder(ILoggerFactory logger, AssembleContext context) +public class AssemblerBuilder(ILoggerFactory logger, AssembleContext context, GlobalNavigation globalNavigation) { - private readonly ILogger _logger = logger.CreateLogger(); - public async Task BuildAllAsync(IReadOnlyCollection checkouts, PublishEnvironment environment, Cancel ctx) { var crossLinkFetcher = new AssemblerCrossLinkFetcher(logger, context.Configuration); - var uriResolver = new PublishEnvironmentUriResolver(context.Configuration, environment); + var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, environment); var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, uriResolver); + if (context.OutputDirectory.Exists) + context.OutputDirectory.Delete(true); + context.OutputDirectory.Create(); + foreach (var checkout in checkouts) { + if (checkout.Repository.Skip) + { + context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its marked as skip in configuration"); + continue; + } + try { await BuildAsync(checkout, environment, crossLinkResolver, ctx); } catch (Exception e) when (e.Message.Contains("Can not locate docset.yml file in")) { - // TODO: we should only ignore this temporarily while migration is ongoing - _logger.LogWarning("Skipping {Checkout} as its not yet been migrated to V3", checkout.Directory.FullName); + context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its not yet been migrated to V3"); } catch (Exception e) { @@ -38,31 +46,26 @@ public async Task BuildAllAsync(IReadOnlyCollection checkouts, Publish throw; } } + + context.Collector.Channel.TryComplete(); + await context.Collector.StopAsync(ctx); } private async Task BuildAsync(Checkout checkout, PublishEnvironment environment, CrossLinkResolver crossLinkResolver, Cancel ctx) { var path = checkout.Directory.FullName; - var localPathPrefix = checkout.Repository.PathPrefix; - var pathPrefix = (environment.PathPrefix, localPathPrefix) switch - { - (null or "", null or "") => null, - (null or "", _) => localPathPrefix, - (_, null or "") => environment.PathPrefix, - var (globalPrefix, docsetPrefix) => $"{globalPrefix}/{docsetPrefix}" - }; - var output = localPathPrefix != null ? Path.Combine(context.OutputDirectory.FullName, localPathPrefix) : context.OutputDirectory.FullName; + var output = environment.PathPrefix != null ? Path.Combine(context.OutputDirectory.FullName, environment.PathPrefix) : context.OutputDirectory.FullName; var buildContext = new BuildContext(context.Collector, context.ReadFileSystem, context.WriteFileSystem, path, output) { - StaticUrlPathPrefix = environment.PathPrefix, - UrlPathPrefix = pathPrefix, - Force = true, - AllowIndexing = environment.AllowIndexing + UrlPathPrefix = environment.PathPrefix, + Force = false, + AllowIndexing = environment.AllowIndexing, + SkipMetadata = true }; var set = new DocumentationSet(buildContext, logger, crossLinkResolver); - var generator = new DocumentationGenerator(set, logger); + var generator = new DocumentationGenerator(set, logger, globalNavigation); await generator.GenerateAll(ctx); } } diff --git a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs index b5e30c720..4f9ac338f 100644 --- a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs +++ b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs @@ -21,10 +21,12 @@ public override async Task Fetch() foreach (var repository in repositories) { - if (repository.Skip) - continue; var repositoryName = repository.Name; _ = declaredRepositories.Add(repositoryName); + + if (repository.Skip) + continue; + var linkReference = await Fetch(repositoryName); linkReferences.Add(repositoryName, linkReference); var linkIndexReference = await GetLinkIndexEntry(repositoryName); @@ -36,7 +38,7 @@ public override async Task Fetch() DeclaredRepositories = declaredRepositories, LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), LinkReferences = linkReferences.ToFrozenDictionary(), - FromConfiguration = true + FromConfiguration = false }; } } diff --git a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs index 4d0595842..75d9f6043 100644 --- a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs +++ b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs @@ -4,50 +4,45 @@ using System.Collections.Frozen; using Documentation.Assembler.Configuration; +using Documentation.Assembler.Navigation; using Elastic.Markdown.CrossLinks; namespace Documentation.Assembler.Building; public class PublishEnvironmentUriResolver : IUriEnvironmentResolver { + private readonly GlobalNavigation _globalNavigation; private Uri BaseUri { get; } private PublishEnvironment PublishEnvironment { get; } - private PreviewEnvironmentUriResolver PreviewResolver { get; } + private IsolatedBuildEnvironmentUriResolver IsolatedBuildResolver { get; } - private FrozenDictionary AllRepositories { get; } - - public PublishEnvironmentUriResolver(AssemblyConfiguration configuration, PublishEnvironment environment) + public PublishEnvironmentUriResolver(GlobalNavigation globalNavigation, PublishEnvironment environment) { + _globalNavigation = globalNavigation; if (!Uri.TryCreate(environment.Uri, UriKind.Absolute, out var uri)) throw new Exception($"Could not parse uri {environment.Uri} in environment {environment}"); BaseUri = uri; PublishEnvironment = environment; - PreviewResolver = new PreviewEnvironmentUriResolver(); - AllRepositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]) - .ToFrozenDictionary(e => e.Name, e => e); - RepositoryLookup = AllRepositories.GetAlternateLookup>(); + IsolatedBuildResolver = new IsolatedBuildEnvironmentUriResolver(); } - private FrozenDictionary.AlternateLookup> RepositoryLookup { get; } - public Uri Resolve(Uri crossLinkUri, string path) { + // TODO Maybe not needed if (PublishEnvironment.Name == "preview") - return PreviewResolver.Resolve(crossLinkUri, path); + return IsolatedBuildResolver.Resolve(crossLinkUri, path); - var repositoryPath = crossLinkUri.Scheme; - if (RepositoryLookup.TryGetValue(crossLinkUri.Scheme, out var repository)) - repositoryPath = repository.PathPrefix; + var subPath = _globalNavigation.GetSubPath(crossLinkUri, ref path); - var fullPath = (PublishEnvironment.PathPrefix, repositoryPath) switch + var fullPath = (PublishEnvironment.PathPrefix, subPath) switch { (null or "", null or "") => path, - (null or "", var p) => $"{p}/{path}", - (var p, null or "") => $"{p}/{path}", - var (p, pp) => $"{p}/{pp}/{path}" + (null or "", var p) => $"{p}/{path.TrimStart('/')}", + (var p, null or "") => $"{p}/{path.TrimStart('/')}", + var (p, pp) => $"{p}/{pp}/{path.TrimStart('/')}" }; return new Uri(BaseUri, fullPath); diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 59d6c3e86..1579e8a8b 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -7,6 +7,7 @@ using Actions.Core.Services; using ConsoleAppFramework; using Documentation.Assembler.Building; +using Documentation.Assembler.Navigation; using Documentation.Assembler.Sourcing; using Elastic.Documentation.Tooling.Diagnostics.Console; using Microsoft.Extensions.Logging; @@ -62,7 +63,11 @@ public async Task BuildAll( var githubEnvironmentInput = githubActionsService.GetInput("environment"); environment ??= !string.IsNullOrEmpty(githubEnvironmentInput) ? githubEnvironmentInput : "dev"; - await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService); + await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService) + { + NoHints = true + }; + _ = collector.StartAsync(ctx); var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null) @@ -79,7 +84,10 @@ public async Task BuildAll( if (checkouts.Length == 0) throw new Exception("No checkouts found"); - var builder = new AssemblerBuilder(logger, assembleContext); + var navigationFile = GlobalNavigationFile.Deserialize(assembleContext); + var globalNavigation = new GlobalNavigation(assembleContext, navigationFile, checkouts); + + var builder = new AssemblerBuilder(logger, assembleContext, globalNavigation); await builder.BuildAllAsync(checkouts, env, ctx); if (strict ?? false) diff --git a/src/docs-assembler/Configuration/AssemblyConfiguration.cs b/src/docs-assembler/Configuration/AssemblyConfiguration.cs index 73c39734b..e394aee68 100644 --- a/src/docs-assembler/Configuration/AssemblyConfiguration.cs +++ b/src/docs-assembler/Configuration/AssemblyConfiguration.cs @@ -6,20 +6,13 @@ namespace Documentation.Assembler.Configuration; -[YamlStaticContext] -[YamlSerializable(typeof(AssemblyConfiguration))] -[YamlSerializable(typeof(Repository))] -[YamlSerializable(typeof(NarrativeRepository))] -[YamlSerializable(typeof(PublishEnvironment))] -public partial class YamlStaticContext; - public record AssemblyConfiguration { public static AssemblyConfiguration Deserialize(string yaml) { var input = new StringReader(yaml); - var deserializer = new StaticDeserializerBuilder(new YamlStaticContext()) + var deserializer = new StaticDeserializerBuilder(new Assembler.YamlStaticContext()) .IgnoreUnmatchedProperties() .Build(); diff --git a/src/docs-assembler/Configuration/Repository.cs b/src/docs-assembler/Configuration/Repository.cs index 68e9aaa5b..46ad8462f 100644 --- a/src/docs-assembler/Configuration/Repository.cs +++ b/src/docs-assembler/Configuration/Repository.cs @@ -10,7 +10,6 @@ public record NarrativeRepository : Repository { public static string RepositoryName { get; } = "docs-content"; public override string Name { get; set; } = RepositoryName; - public override string? PathPrefix { get; set; } } public record Repository @@ -27,14 +26,6 @@ public record Repository [YamlMember(Alias = "checkout_strategy")] public string CheckoutStrategy { get; set; } = "partial"; - private string? _pathPrefix; - [YamlMember(Alias = "path_prefix")] - public virtual string? PathPrefix - { - get => _pathPrefix ?? $"reference/{Name}"; - set => _pathPrefix = value; - } - [YamlMember(Alias = "skip")] public bool Skip { get; set; } } diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs new file mode 100644 index 000000000..97aebcc85 --- /dev/null +++ b/src/docs-assembler/Navigation/GlobalNavigation.cs @@ -0,0 +1,152 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using System.Collections.Immutable; +using System.IO.Abstractions; +using Documentation.Assembler.Configuration; +using Documentation.Assembler.Sourcing; +using Elastic.Markdown; +using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; + +namespace Documentation.Assembler.Navigation; + +public record GlobalNavigation : IDocumentationFileOutputProvider +{ + private readonly AssembleContext _context; + private readonly FrozenDictionary.AlternateLookup> _checkoutsLookup; + private readonly FrozenDictionary.AlternateLookup> _repoConfigLookup; + private readonly FrozenDictionary.AlternateLookup> _tocLookup; + + private FrozenDictionary ConfiguredRepositories { get; } + private FrozenDictionary IndexedTableOfContents { get; } + + public GlobalNavigationFile NavigationConfiguration { get; init; } + + private FrozenDictionary Checkouts { get; init; } + + private ImmutableSortedSet TableOfContentsPrefixes { get; } + + public GlobalNavigation(AssembleContext context, GlobalNavigationFile navigationConfiguration, Checkout[] checkouts) + { + _context = context; + NavigationConfiguration = navigationConfiguration; + Checkouts = checkouts.ToDictionary(c => c.Repository.Name, c => c).ToFrozenDictionary(); + _checkoutsLookup = Checkouts.GetAlternateLookup>(); + + var configuration = context.Configuration; + ConfiguredRepositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]) + .ToFrozenDictionary(e => e.Name, e => e); + _repoConfigLookup = ConfiguredRepositories.GetAlternateLookup>(); + + IndexedTableOfContents = navigationConfiguration.IndexedTableOfContents; + _tocLookup = IndexedTableOfContents.GetAlternateLookup>(); + TableOfContentsPrefixes = navigationConfiguration.IndexedTableOfContents.Keys.OrderByDescending(k => k.Length).ToImmutableSortedSet(); + } + + public string GetSubPath(Uri crossLinkUri, ref string path) + { + if (!_checkoutsLookup.TryGetValue(crossLinkUri.Scheme, out _)) + { + _context.Collector.EmitError(_context.ConfigurationPath, + !_repoConfigLookup.TryGetValue(crossLinkUri.Scheme, out _) + ? $"Repository: '{crossLinkUri.Scheme}' is not defined in assembler.yml" + : $"Unable to find checkout for repository: {crossLinkUri.Scheme}" + ); + } + + var lookup = crossLinkUri.ToString().AsSpan(); + if (lookup.EndsWith(".md", StringComparison.Ordinal)) + lookup = lookup[..^3]; + + // temporary fix only spotted two instances of this: + // Error: Unable to find defined toc for url: docs-content:///manage-data/ingest/transform-enrich/set-up-an-enrich-processor.md + // Error: Unable to find defined toc for url: kibana:///reference/configuration-reference.md + if (lookup.IndexOf(":///") >= 0) + lookup = lookup.ToString().Replace(":///", "://").AsSpan(); + + string? match = null; + foreach (var prefix in TableOfContentsPrefixes) + { + if (!lookup.StartsWith(prefix, StringComparison.Ordinal)) + continue; + match = prefix; + break; + } + + if (match is null || !_tocLookup.TryGetValue(match, out var toc)) + { + //TODO remove + if (crossLinkUri.Scheme != "asciidocalypse") + _context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for url: {crossLinkUri}"); + return $"reference/{crossLinkUri.Scheme}"; + } + + path = path.AsSpan().TrimStart(toc.SourcePrefix).ToString(); + + return toc.PathPrefix; + } + + public IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath) + { + if (relativePath.StartsWith("_static/", StringComparison.Ordinal)) + return defaultOutputFile; + + var outputDirectory = documentationSet.OutputDirectory; + var fs = defaultOutputFile.FileSystem; + + var l = $"{documentationSet.Name}://{relativePath.TrimStart('/')}"; + var lookup = l.AsSpan(); + if (lookup.StartsWith("docs-content://serverless/", StringComparison.Ordinal)) + return null; + if (lookup.StartsWith("eland://sphinx/", StringComparison.Ordinal)) + return null; + if (lookup.StartsWith("elasticsearch-py://sphinx/", StringComparison.Ordinal)) + return null; + if (lookup.StartsWith("elastic-serverless-forwarder://", StringComparison.Ordinal) && lookup.EndsWith(".png")) + return null; + + //allow files at root for `docs-content` (index.md 404.md) + if (lookup.StartsWith("docs-content://") && !relativePath.Contains('/')) + return defaultOutputFile; + + string? match = null; + foreach (var prefix in TableOfContentsPrefixes) + { + if (!lookup.StartsWith(prefix, StringComparison.Ordinal)) + continue; + match = prefix; + break; + } + + if (match is null || !_tocLookup.TryGetValue(match, out var toc)) + { + if (relativePath.StartsWith("raw-migrated-files/", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("images/", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("examples/", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("docset.yml", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("doc_examples", StringComparison.Ordinal)) + return null; + if (relativePath.EndsWith(".asciidoc", StringComparison.Ordinal)) + return null; + + var fallBack = fs.Path.Combine(outputDirectory.FullName, "_failed", documentationSet.Name, relativePath); + _context.Collector.EmitError(_context.NavigationPath, $"No toc for output path: '{lookup}' falling back to: '{fallBack}'"); + return fs.FileInfo.New(fallBack); + } + + var newPath = relativePath.AsSpan().TrimStart(toc.SourcePrefix.TrimEnd('/')).TrimStart('/').ToString(); + var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, newPath); + if (path.Contains("deploy-manage")) + { + } + + return fs.FileInfo.New(path); + } +} diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs new file mode 100644 index 000000000..7c5dd315e --- /dev/null +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -0,0 +1,168 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using Documentation.Assembler.Configuration; +using Elastic.Markdown.IO.Configuration; +using YamlDotNet.RepresentationModel; + +namespace Documentation.Assembler.Navigation; + +public record TableOfContentsReference +{ + public required Uri Source { get; init; } + public required string SourcePrefix { get; init; } + public required string PathPrefix { get; init; } + public required IReadOnlyCollection Children { get; init; } +} + +public record GlobalNavigationFile +{ + public IReadOnlyCollection TableOfContents { get; init; } = []; + + public FrozenDictionary IndexedTableOfContents { get; init; } = + new Dictionary().ToFrozenDictionary(); + + public static GlobalNavigationFile Deserialize(AssembleContext context) + { + var globalConfig = new GlobalNavigationFile(); + var reader = new YamlStreamReader(context.NavigationPath, context.Collector); + try + { + foreach (var entry in reader.Read()) + { + switch (entry.Key) + { + case "toc": + var toc = ReadChildren(reader, entry.Entry, null); + var indexed = toc + .SelectMany(YieldAll) + .ToDictionary(t => t.Source.ToString(), t => t) + .ToFrozenDictionary(); + globalConfig = globalConfig with + { + TableOfContents = toc, + IndexedTableOfContents = indexed + }; + break; + } + } + } + catch (Exception e) + { + reader.EmitError("Could not load docset.yml", e); + throw; + } + + return globalConfig; + } + + private static IEnumerable YieldAll(TableOfContentsReference toc) + { + yield return toc; + foreach (var tocEntry in toc.Children) + { + foreach (var child in YieldAll(tocEntry)) + yield return child; + } + } + + private static IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyValuePair entry, string? parent) + { + var entries = new List(); + if (entry.Key is not YamlScalarNode { Value: { } key } scalarKey) + { + reader.EmitWarning($"key '{entry.Key}' is not string"); + return []; + } + + if (entry.Value is not YamlSequenceNode sequence) + { + reader.EmitWarning($"'{scalarKey.Value}' is not an array"); + return []; + } + + foreach (var tocEntry in sequence.Children.OfType()) + { + var child = ReadChild(reader, tocEntry, parent); + if (child is not null) + entries.Add(child); + } + + return entries; + } + + private static TableOfContentsReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, string? parent) + { + string? repository = null; + string? source = null; + string? pathPrefix = null; + IReadOnlyCollection? children = null; + foreach (var entry in tocEntry.Children) + { + var key = ((YamlScalarNode)entry.Key).Value; + switch (key) + { + case "toc": + source = reader.ReadString(entry); + if (source.AsSpan().IndexOf("://") == -1) + { + parent = source; + pathPrefix = source; + source = $"{NarrativeRepository.RepositoryName}://{source}"; + } + + break; + case "repo": + repository = reader.ReadString(entry); + break; + case "path_prefix": + pathPrefix = reader.ReadString(entry); + break; + case "children": + if (source is null && pathPrefix is null) + { + reader.EmitWarning("toc entry has no toc or path_prefix defined"); + continue; + } + + children = ReadChildren(reader, entry, parent); + break; + } + } + + if (repository is not null) + { + if (source is not null) + reader.EmitError($"toc config defines 'repo' can not be combined with 'toc': {source}", tocEntry); + if (children is not null) + reader.EmitError($"toc config defines 'repo' can not be combined with 'children'", tocEntry); + pathPrefix = string.Join("/", [parent, repository]); + source = $"{repository}://{parent}"; + } + + if (source is null) + return null; + + if (!Uri.TryCreate(source, UriKind.Absolute, out var sourceUri)) + { + reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry); + return null; + } + + var sourcePrefix = $"{sourceUri.Host}/{sourceUri.AbsolutePath.TrimStart('/')}"; + if (string.IsNullOrEmpty(pathPrefix)) + reader.EmitError($"Path prefix is not defined for: {source}, falling back to {sourcePrefix} which may be incorrect", tocEntry); + + pathPrefix ??= sourcePrefix; + + return new TableOfContentsReference + { + Source = sourceUri, + SourcePrefix = sourcePrefix, + Children = children ?? [], + PathPrefix = pathPrefix + }; + } +} diff --git a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 80aa4b515..feaf820b0 100644 --- a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -27,12 +27,12 @@ public IReadOnlyCollection GetAll() foreach (var repo in repositories) { var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, repo.Name)); - var head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + //var head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); var checkout = new Checkout { Repository = repo, Directory = checkoutFolder, - HeadReference = head + HeadReference = Guid.NewGuid().ToString("N") }; checkouts.Add(checkout); } @@ -44,13 +44,6 @@ public async Task> AcquireAllLatest(Cancel ctx = d var dict = new ConcurrentDictionary(); var checkouts = new ConcurrentBag(); - if (context.OutputDirectory.Exists) - { - _logger.LogInformation("Cleaning output directory: {OutputDirectory}", context.OutputDirectory.FullName); - context.OutputDirectory.Delete(true); - } - - _logger.LogInformation("Cloning narrative content: {Repository}", NarrativeRepository.RepositoryName); var checkout = CloneOrUpdateRepository(Configuration.Narrative, NarrativeRepository.RepositoryName, dict); checkouts.Add(checkout); @@ -90,7 +83,8 @@ private Checkout CloneOrUpdateRepository(Repository repository, string name, Con _logger.LogInformation("Pull: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath); // --allow-unrelated-histories due to shallow clones not finding a common ancestor ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff"); - head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + //head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + head = Guid.NewGuid().ToString("N"); } else { @@ -111,7 +105,8 @@ private Checkout CloneOrUpdateRepository(Repository repository, string name, Con ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone"); ExecIn(checkoutFolder, "git", "checkout", repository.CurrentBranch); ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs"); - head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + //head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + head = Guid.NewGuid().ToString("N"); } } diff --git a/src/docs-assembler/YamlStaticContext.cs b/src/docs-assembler/YamlStaticContext.cs new file mode 100644 index 000000000..fd57edc10 --- /dev/null +++ b/src/docs-assembler/YamlStaticContext.cs @@ -0,0 +1,15 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Documentation.Assembler.Configuration; +using YamlDotNet.Serialization; + +namespace Documentation.Assembler; + +[YamlStaticContext] +[YamlSerializable(typeof(AssemblyConfiguration))] +[YamlSerializable(typeof(Repository))] +[YamlSerializable(typeof(NarrativeRepository))] +[YamlSerializable(typeof(PublishEnvironment))] +public partial class YamlStaticContext; diff --git a/src/docs-assembler/assembler.yml b/src/docs-assembler/assembler.yml index f65c5df7e..111108acc 100644 --- a/src/docs-assembler/assembler.yml +++ b/src/docs-assembler/assembler.yml @@ -1,6 +1,6 @@ environments: production: - uri: https://elastic.co + uri: https://www.elastic.co path_prefix: docs allow_indexing: false staging: @@ -15,6 +15,9 @@ environments: narrative: checkout_strategy: full references: + #TODO TEMPORARY REMOVE + asciidocalypse: + skip: true apm-agent-android: apm-agent-dotnet: apm-agent-go: @@ -77,6 +80,7 @@ references: logstash-docs: skip: true logstash: + skip: true search-ui: integration-docs: skip: true diff --git a/src/docs-assembler/docs-assembler.csproj b/src/docs-assembler/docs-assembler.csproj index 733f18395..ec1ec216d 100644 --- a/src/docs-assembler/docs-assembler.csproj +++ b/src/docs-assembler/docs-assembler.csproj @@ -31,5 +31,6 @@ + diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml new file mode 100644 index 000000000..e1731048f --- /dev/null +++ b/src/docs-assembler/navigation.yml @@ -0,0 +1,514 @@ +############################################# +# KEY FOR REFERENCE SECTION +# ✅ = toc should be ready to go! +# 🔜 = ready in a PR/branch, +# but needs to be approved and merged +# 📝 = no PR started yet +############################################# + +toc: + ############# + # NARRATIVE # + ############# + - toc: get-started + - toc: solutions + - toc: manage-data + - toc: explore-analyze + - toc: deploy-manage + - toc: cloud-account + - toc: troubleshoot + + ########## + # EXTEND # + ########## + # I didn't touch this section (yet?) + - toc: extend + children: + - toc: kibana://extend + path_prefix: extend/kibana + - toc: logstash://extend + path_prefix: extend/logstash + - toc: elasticsearch://extend + path_prefix: extend/elasticsearch + - toc: integrations://extend + path_prefix: extend/integrations + #- toc: beats://extend + + ################# + # RELEASE NOTES # + ################# + # I didn't touch this section (yet?) + - toc: release-notes + children: + # repo is short for + # - toc: :// + # path_prefix: / + - repo: asciidocalypse + - repo: apm-agent-android + - repo: apm-agent-dotnet + - repo: apm-agent-go + - repo: apm-agent-ios + - repo: apm-agent-java + - repo: apm-agent-nodejs + - repo: apm-agent-php + - repo: apm-agent-python + - repo: apm-agent-ruby + - repo: apm-agent-rum-js + - repo: apm-aws-lambda + - repo: apm-k8s-attacher + - repo: beats + - repo: cloud-on-k8s + - repo: cloud + - repo: curator + - repo: ecctl + - repo: ecs-dotnet + - repo: ecs-logging-go-logrus + - repo: ecs-logging-go-zap + - repo: ecs-logging-go-zerolog + - repo: ecs-logging-java + - repo: ecs-logging-nodejs + - repo: ecs-logging-php + - repo: ecs-logging-python + - repo: ecs-logging-ruby + - repo: ecs-logging + - repo: ecs + - repo: eland + - repo: elastic-serverless-forwarder + - repo: elasticsearch-hadoop + - repo: elasticsearch-java + - repo: elasticsearch-js + - repo: elasticsearch-net + - repo: elasticsearch-php + - repo: elasticsearch-py + - repo: elasticsearch-rs + - repo: elasticsearch-ruby + - repo: elasticsearch + - repo: go-elasticsearch + - repo: integrations + - repo: kibana + - repo: logstash-docs + - repo: logstash + - repo: search-ui + - repo: integration-docs + - repo: security-docs + + ############# + # REFERENCE # + ############# + - toc: reference + children: + # Security + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/security/toc.yml + - toc: docs-content://reference/security + path_prefix: reference/security + # Children include: Endpoint command reference, Elastic Defend, + # Fields and object schemas + children: + # 📝 TO DO: I don't remember the repo name + # 📝 TO DO: toc.yml needs to be created + - toc: that-other-sec-repo://reference + path_prefix: reference/security/prebuilt-rules + # Children include the entire AsciiDoc book + + # Observability + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/observability/toc.yml + - toc: docs-content://reference/observability + path_prefix: reference/observability + # Children include: Fields and object schemas, Elastic Entity Model, + # Infrastructure app fields + + # Search + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/search/toc.yml + - toc: docs-content://reference/search + path_prefix: reference/search + children: + # Search UI + # ✅ https://github.com/elastic/search-ui/blob/main/docs/reference/toc.yml + - toc: search-ui://reference + path_prefix: reference/search-ui + # Children include the entire AsciiDoc book + + # Elasticsearch and index management + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/elasticsearch/toc.yml + - toc: docs-content://reference/elasticsearch + path_prefix: reference/elasticsearch + children: + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/elasticsearch/toc.yml + - toc: elasticsearch://reference/elasticsearch + path_prefix: reference/elasticsearch + # Children include: Configuration, JVM settings, Roles, + # Elasticsearch privileges, Index settings, Index lifecycle actions, + # REST APIs, Mapping, Elasticsearch audit events, Command line tools + + # Curator + # ✅ https://github.com/elastic/curator/blob/master/docs/reference/toc.yml + - toc: curator://reference + path_prefix: reference/elasticsearch/curator + # Children include the entire AsciiDoc book + + # Clients + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/elasticsearch-clients/toc.yml + - toc: docs-content://reference/elasticsearch/clients + path_prefix: reference/elasticsearch/clients + children: + + # Eland + # ✅ https://github.com/elastic/eland/blob/main/docs/reference/toc.yml + - toc: eland://reference + path_prefix: reference/elasticsearch/clients/eland + # Children include the entire AsciiDoc book + + # Go + # ✅ https://github.com/elastic/go-elasticsearch/blob/main/docs/reference/toc.yml + - toc: go-elasticsearch://reference + path_prefix: reference/elasticsearch/clients/go + # Children include the entire AsciiDoc book + + # Java + # ✅ https://github.com/elastic/elasticsearch-java/blob/main/docs/reference/toc.yml + - toc: elasticsearch-java://reference + path_prefix: reference/elasticsearch/clients/java + # Children include the entire AsciiDoc book + + # JavaScript + # ✅ https://github.com/elastic/elasticsearch-js/blob/main/docs/reference/toc.yml + - toc: elasticsearch-js://reference + path_prefix: reference/elasticsearch/clients/js + # Children include the entire AsciiDoc book + + # .NET + # ✅ https://github.com/elastic/elasticsearch-net/blob/main/docs/reference/toc.yml + - toc: elasticsearch-net://reference + path_prefix: reference/elasticsearch/clients/net + # Children include the entire AsciiDoc book + + # PHP + # ✅ https://github.com/elastic/elasticsearch-php/blob/main/docs/reference/toc.yml + - toc: elasticsearch-php://reference + path_prefix: reference/elasticsearch/clients/php + # Children include the entire AsciiDoc book + + # Python + # ✅ https://github.com/elastic/elasticsearch-py/blob/main/docs/reference/toc.yml + - toc: elasticsearch-py://reference + path_prefix: reference/elasticsearch/clients/py + # Children include the entire AsciiDoc book + + # Ruby + # ✅ https://github.com/elastic/elasticsearch-ruby/blob/main/docs/reference/toc.yml + - toc: elasticsearch-ruby://reference + path_prefix: reference/elasticsearch/clients/ruby + # Children include the entire AsciiDoc book + + # Rust + # ✅ https://github.com/elastic/elasticsearch-rs/blob/main/docs/reference/toc.yml + - toc: elasticsearch-rs://reference + path_prefix: reference/elasticsearch/clients/rs + # Children include the entire AsciiDoc book + + # Community-contributed clients + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/community-contributed/toc.yml + - toc: elasticsearch://reference/community-contributed + path_prefix: reference/elasticsearch/clients/community-contributed + + # Ingestion tools + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/ingestion-tools/toc.yml + - toc: docs-content://reference/ingestion-tools + path_prefix: reference/ingestion-tools + children: + # I don't know (TODO discuss with Colleen) + - toc: elasticsearch://reference/ingestion-tools/enrich-processor + path_prefix: reference/elasticsearch/enrich-processor + - toc: elasticsearch://reference/ingestion-tools/search-connectors + path_prefix: reference/elasticsearch/search-connectors + - toc: elasticsearch://reference/data-analysis + path_prefix: reference/elasticsearch/data-analysis + - toc: security-docs://reference/prebuilt-rules + path_prefix: reference/prebuilt-rules + - toc: elasticsearch://reference + path_prefix: reference/elasticsearch + + # Fleet and Elastic Agent + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/fleet/toc.yml + - toc: docs-content://reference/fleet + path_prefix: reference/fleet + # Children include the entire AsciiDoc book + + # APM + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/apm/toc.yml + - toc: docs-content://reference/apm + path_prefix: reference/apm + # Children include: APM settings, APM settings for Elastic Cloud, + # APM settings for Elastic Cloud Enterprise + children: + # APM Attacher for Kubernetes + # ✅ https://github.com/elastic/apm-k8s-attacher/blob/main/docs/reference/toc.yml + - toc: apm-k8s-attacher://reference + path_prefix: reference/apm/k8s-attacher + # Children include the entire AsciiDoc book + + # APM Architecture for AWS Lambda + # ✅ https://github.com/elastic/apm-aws-lambda/blob/main/docs/reference/toc.yml + - toc: apm-aws-lambda://reference + path_prefix: reference/apm/aws-lambda + # Children include the entire AsciiDoc book + + # APM agents + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/apm-agents/toc.yml + - toc: docs-content://reference/ingestion-tools/apm/agents + path_prefix: reference/apm/agents + children: + # APM Android agent + # ✅ https://github.com/elastic/apm-agent-android/blob/main/docs/reference/toc.yml + - toc: apm-agent-android://reference + path_prefix: reference/apm/agents/android + # Children include the entire AsciiDoc book + + # APM .NET agent + # ✅ https://github.com/elastic/apm-agent-dotnet/blob/main/docs/reference/toc.yml + - toc: apm-agent-dotnet://reference + path_prefix: reference/apm/agents/dotnet + # Children include the entire AsciiDoc book + + # APM Go agent + # ✅ https://github.com/elastic/apm-agent-go/blob/main/docs/reference/toc.yml + - toc: apm-agent-go://reference + path_prefix: reference/apm/agents/go + # Children include the entire AsciiDoc book + + # APM iOS agent + # ✅ https://github.com/elastic/apm-agent-ios/blob/main/docs/reference/toc.yml + - toc: apm-agent-ios://reference + path_prefix: reference/apm/agents/ios + # Children include the entire AsciiDoc book + + # APM Java agent + # ✅ https://github.com/elastic/apm-agent-java/blob/main/docs/reference/toc.yml + - toc: apm-agent-java://reference + path_prefix: reference/apm/agents/java + # Children include the entire AsciiDoc book + + # APM Node.js agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-nodejs/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-nodejs://reference + path_prefix: reference/apm/agents/nodejs + # Children include the entire AsciiDoc book + + # APM PHP agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-php/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-php://reference + path_prefix: reference/apm/agents/php + # Children include the entire AsciiDoc book + + # APM Python agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-python/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-python://reference + path_prefix: reference/apm/agents/python + # Children include the entire AsciiDoc book + + # APM Ruby agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-ruby/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-ruby://reference + path_prefix: reference/apm/agents/ruby + # Children include the entire AsciiDoc book + + # APM RUM JavaScript agent + # ✅ https://github.com/elastic/apm-agent-rum-js/blob/main/docs/reference/toc.yml + - toc: apm-agent-rum-js://reference + path_prefix: reference/apm/agents/rum-js + # Children include the entire AsciiDoc book + + # Beats + # 🔜 https://github.com/colleenmcginnis/beats/blob/migrate-docs/docs/reference/toc.yml + - toc: beats://reference + path_prefix: reference/beats + # Children include all entire AsciiDoc books: Beats, Auditbeat, + # Filebeat, Heartbeat, Metricbeat, Packetbeat, Winlogbeat, + # Elastic logging plugin for Docker + + # Processor reference + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/enrich-processor/toc.yml + - toc: elasticsearch://reference/enrich-processor + path_prefix: reference/enrich-processor + + # Logstash + # ✅ https://github.com/elastic/logstash/blob/main/docs/reference/toc.yml + - toc: logstash://reference + path_prefix: reference/logstash + # Children include the entire AsciiDoc book + + # Logstash plugins + # 📝 TO DO: Migrate all files and create toc.yml + - toc: logstash-docs://reference + path_prefix: reference/logstash/plugins + # Children include the entire AsciiDoc book + + # Elastic Serverless Forwarder for AWS + # ✅ https://github.com/elastic/elastic-serverless-forwarder/blob/main/docs/reference/toc.yml + - toc: elastic-serverless-forwarder://reference + path_prefix: reference/elastic-serverless-forwarder + # Children include the entire AsciiDoc book + + # Search connectors + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/search-connectors/toc.yml + - toc: elasticsearch://reference/search-connectors + path_prefix: reference/search-connectors + + # Elasticsearch Hadoop + # ✅ https://github.com/elastic/elasticsearch-hadoop/blob/main/docs/reference/toc.yml + - toc: elasticsearch-hadoop://reference + path_prefix: reference/elasticsearch-hadoop + # Children include the entire AsciiDoc book + + # Elastic Integrations + # 📝 TO DO: Waiting on integration devs to support 9.0 in all integrations + - toc: integration-docs://reference + path_prefix: reference/integrations + # Children include the entire AsciiDoc book + + # Kibana + # ✅ https://github.com/elastic/kibana/blob/main/docs/reference/toc.yml + - toc: kibana://reference + path_prefix: reference/kibana + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Elasticsearch plugins + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/elasticsearch-plugins/toc.yml + - toc: elasticsearch://reference/elasticsearch-plugins + path_prefix: reference/elasticsearch-plugins + + # Query languages + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/query-languages/toc.yml + - toc: elasticsearch://reference/query-languages + path_prefix: reference/query-languages + + # Scripting languages + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/scripting-languages/toc.yml + - toc: elasticsearch://reference/scripting-languages + path_prefix: reference/scripting-languages + + # ECS + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/ecs/toc.yml + - toc: docs-content://reference/ecs + path_prefix: reference/ecs + children: + # ECS reference + # ✅ https://github.com/elastic/ecs/blob/main/docs/reference/toc.yml + - toc: ecs://reference + path_prefix: reference/ecs + # Children include the entire AsciiDoc book + + # ECS logging libraries + # ✅ https://github.com/elastic/ecs-logging/blob/main/docs/reference/toc.yml + - toc: ecs-logging://reference + path_prefix: reference/ecs/logging + children: + # ECS Logging .NET + # ✅ https://github.com/elastic/ecs-dotnet/blob/main/docs/reference/toc.yml + - toc: ecs-dotnet://reference + path_prefix: reference/ecs/logging/dotnet + # Children include the entire AsciiDoc book + + # ECS Logging Go (Logrus) + # ✅ https://github.com/elastic/ecs-logging-go-logrus/blob/main/docs/reference/toc.yml + - toc: ecs-logging-go-logrus://reference + path_prefix: reference/ecs/logging/go-logrus + # Children include the entire AsciiDoc book + + # ECS Logging Go (Zap) + # ✅ https://github.com/elastic/ecs-logging-go-zap/blob/main/docs/reference/toc.yml + - toc: ecs-logging-go-zap://reference + path_prefix: reference/ecs/logging/go-zap + # Children include the entire AsciiDoc book + + # ECS Logging Go (Zerolog) + # ✅ https://github.com/elastic/ecs-logging-go-zerolog/blob/main/docs/reference/toc.yml + - toc: ecs-logging-go-zerolog://reference + path_prefix: reference/ecs/logging/go-zerolog + # Children include the entire AsciiDoc book + + # ECS Logging Java + # ✅ https://github.com/elastic/ecs-logging-java/blob/main/docs/reference/toc.yml + - toc: ecs-logging-java://reference + path_prefix: reference/ecs/logging/java + # Children include the entire AsciiDoc book + + # ECS Logging Node.js + # ✅ https://github.com/elastic/ecs-logging-nodejs/blob/main/docs/reference/toc.yml + - toc: ecs-logging-nodejs://reference + path_prefix: reference/ecs/logging/nodejs + # Children include the entire AsciiDoc book + + # ECS Logging PHP + # ✅ https://github.com/elastic/ecs-logging-php/blob/main/docs/reference/toc.yml + - toc: ecs-logging-php://reference + path_prefix: reference/ecs/logging/php + # Children include the entire AsciiDoc book + + # ECS Logging Python + # ✅ https://github.com/elastic/ecs-logging-python/blob/main/docs/reference/toc.yml + - toc: ecs-logging-python://reference + path_prefix: reference/ecs/logging/python + # Children include the entire AsciiDoc book + + # ECS Logging Ruby + # ✅ https://github.com/elastic/ecs-logging-ruby/blob/main/docs/reference/toc.yml + - toc: ecs-logging-ruby://reference + path_prefix: reference/ecs/logging/ruby + # Children include the entire AsciiDoc book + + # Data analysis + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/data-analysis/toc.yml + - toc: docs-content://reference/data-analysis + path_prefix: reference/data-analysis + # Children include: Supplied configurations, Function reference, + # Metrics reference, Canvas function reference + children: + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/text-analysis/toc.yml + - toc: elasticsearch://reference/text-analysis + path_prefix: reference/text-analysis + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/aggregations/toc.yml + - toc: elasticsearch://reference/aggregations + path_prefix: reference/aggregations + + # Cloud + # 📝 TO DO: toc.yml needs to be created with one file / no children + - toc: docs-content://reference/cloud + path_prefix: reference/cloud + children: + # Elastic Cloud Enterprise + # Elastic Cloud Hosted + # ✅ https://github.com/elastic/cloud/blob/master/docs/reference/toc.yml + - toc: cloud://reference + path_prefix: reference/cloud + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Elastic Cloud on Kubernetes + # ✅ https://github.com/elastic/cloud-on-k8s/blob/main/docs/reference/toc.yml + - toc: cloud-on-k8s://reference + path_prefix: reference/cloud-on-k8s + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Elastic cloud control (ECCTL) + # ✅ https://github.com/elastic/ecctl/blob/master/docs/reference/toc.yml + - toc: ecctl://reference + path_prefix: reference/ecctl + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Glossary + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/glossary/toc.yml + - toc: docs-content://reference/glossary + path_prefix: reference/glossary \ No newline at end of file diff --git a/src/docs-builder/Cli/Commands.cs b/src/docs-builder/Cli/Commands.cs index c7e87c5a7..0f5b03bdd 100644 --- a/src/docs-builder/Cli/Commands.cs +++ b/src/docs-builder/Cli/Commands.cs @@ -133,8 +133,9 @@ public async Task Generate( metadataOnly ??= metaValue; var exporter = metadataOnly.HasValue && metadataOnly.Value ? new NoopDocumentationFileExporter() : null; - var generator = new DocumentationGenerator(set, logger, exporter); + var generator = new DocumentationGenerator(set, logger, null, exporter); await generator.GenerateAll(ctx); + await generator.StopDiagnosticCollection(ctx); if (runningOnCi) await githubActionsService.SetOutputAsync("landing-page-path", set.MarkdownFiles.First().Value.Url); if (bool.TryParse(githubActionsService.GetInput("strict"), out var strictValue) && strictValue) diff --git a/src/docs-builder/Http/StaticWebHost.cs b/src/docs-builder/Http/StaticWebHost.cs index e9a38d04f..0fa63197a 100644 --- a/src/docs-builder/Http/StaticWebHost.cs +++ b/src/docs-builder/Http/StaticWebHost.cs @@ -49,10 +49,7 @@ private void SetUpRoutes() _ = _webApplication.MapGet("/", (Cancel _) => Results.Redirect("docs")); - _ = _webApplication.MapGet("docs/", (Cancel ctx) => - ServeDocumentationFile("index.html", ctx)); - - _ = _webApplication.MapGet("docs/{**slug}", (string slug, Cancel ctx) => + _ = _webApplication.MapGet("{**slug}", (string slug, Cancel ctx) => ServeDocumentationFile(slug, ctx)); } diff --git a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs index e11aadfcf..8d00c2464 100644 --- a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs +++ b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs @@ -15,7 +15,13 @@ public async Task CreatesDefaultOutputDirectory() var logger = new TestLoggerFactory(output); var fileSystem = new MockFileSystem(new Dictionary { - { "docs/docset.yml", new MockFileData("") }, + { "docs/docset.yml", + //language=yaml + new MockFileData(""" +project: test +toc: +- file: index.md +""") }, { "docs/index.md", new MockFileData("test") } }, new MockFileSystemOptions { @@ -27,6 +33,7 @@ public async Task CreatesDefaultOutputDirectory() var generator = new DocumentationGenerator(set, logger); await generator.GenerateAll(TestContext.Current.CancellationToken); + await generator.StopDiagnosticCollection(TestContext.Current.CancellationToken); fileSystem.Directory.Exists(".artifacts").Should().BeTrue(); diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index 9f9cc894b..39b99ca04 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -12,7 +12,7 @@ namespace Elastic.Markdown.Tests; public class TestCrossLinkResolver : ICrossLinkResolver { - private readonly IUriEnvironmentResolver _uriResolver = new PreviewEnvironmentUriResolver(); + private readonly IUriEnvironmentResolver _uriResolver = new IsolatedBuildEnvironmentUriResolver(); private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; private Dictionary LinkReferences { get; } = []; private HashSet DeclaredRepositories { get; } = []; diff --git a/tests/authoring/Framework/Setup.fs b/tests/authoring/Framework/Setup.fs index 7c179b915..36ea48380 100644 --- a/tests/authoring/Framework/Setup.fs +++ b/tests/authoring/Framework/Setup.fs @@ -102,7 +102,7 @@ type Setup = let conversionCollector = TestConversionCollector() let linkResolver = TestCrossLinkResolver(context.Configuration) let set = DocumentationSet(context, logger, linkResolver); - let generator = DocumentationGenerator(set, logger, null, conversionCollector) + let generator = DocumentationGenerator(set, logger, null, null, conversionCollector) let context = { Collector = collector diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index f43202ad6..0e668343a 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -18,7 +18,7 @@ type TestCrossLinkResolver (config: ConfigurationFile) = let references = Dictionary() let declared = HashSet() - let uriResolver = PreviewEnvironmentUriResolver() + let uriResolver = IsolatedBuildEnvironmentUriResolver() member this.LinkReferences = references member this.DeclaredRepositories = declared diff --git a/tests/authoring/Framework/TestValues.fs b/tests/authoring/Framework/TestValues.fs index 7dc33ad07..c68c6c6b2 100644 --- a/tests/authoring/Framework/TestValues.fs +++ b/tests/authoring/Framework/TestValues.fs @@ -99,6 +99,7 @@ and MarkdownTestContext = member this.Bootstrap () = backgroundTask { let! ctx = Async.CancellationToken do! this.Generator.GenerateAll(ctx) + do! this.Generator.StopDiagnosticCollection(ctx) let results = this.ConversionCollector.Results diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs new file mode 100644 index 000000000..094d15ce4 --- /dev/null +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; +using Documentation.Assembler.Building; +using Documentation.Assembler.Configuration; +using Documentation.Assembler.Navigation; +using Documentation.Assembler.Sourcing; +using Elastic.Markdown.Diagnostics; +using FluentAssertions; + +namespace Documentation.Assembler.Tests; + +public class GlobalNavigationTests +{ + [Fact] + public async Task ParsesGlobalNavigation() + { + await using var collector = new DiagnosticsCollector([]); + _ = collector.StartAsync(TestContext.Current.CancellationToken); + + var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null); + var globalNavigation = GlobalNavigationFile.Deserialize(assembleContext); + globalNavigation.TableOfContents.Should().NotBeNull().And.NotBeEmpty(); + var docsContentKeys = globalNavigation.IndexedTableOfContents.Keys + .Where(k => k.StartsWith("docs-content://", StringComparison.Ordinal)).ToArray(); + docsContentKeys.Should().Contain("docs-content://solutions/"); + } + + public static Checkout CreateCheckout(IFileSystem fs, string name) => + new() + { + Repository = new Repository { Name = name, Origin = $"elastic/{name}" }, + HeadReference = Guid.NewGuid().ToString(), + Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", name)) + }; + + [Fact] + public async Task UriResolving() + { + await using var collector = new DiagnosticsCollector([]); + _ = collector.StartAsync(TestContext.Current.CancellationToken); + + var fs = new FileSystem(); + var assembleContext = new AssembleContext(collector, fs, fs, null, null); + var globalNavigationFile = GlobalNavigationFile.Deserialize(assembleContext); + globalNavigationFile.TableOfContents.Should().NotBeNull().And.NotBeEmpty(); + string[] repos = ["docs-content", "curator", "elasticsearch-net", "elasticsearch"]; + var checkouts = repos.Select(r => CreateCheckout(fs, r)).ToArray(); + var globalNavigation = new GlobalNavigation(assembleContext, globalNavigationFile, checkouts); + + var env = assembleContext.Configuration.Environments["production"]; + var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, env); + + // docs-content://reference/apm/something.md - url hasn't changed + var resolvedURi = uriResolver.Resolve(new Uri("docs-content://reference/apm/something.md"), "/reference/apm/something"); + resolvedURi.Should().Be("https://www.elastic.co/docs/reference/apm/something"); + + + resolvedURi = uriResolver.Resolve(new Uri("curator://reference/a/file.md"), "/reference/a/file"); + resolvedURi.Should().Be("https://www.elastic.co/docs/reference/elasticsearch/curator/a/file"); + + resolvedURi = uriResolver.Resolve(new Uri("elasticsearch-net://reference/b/file.md"), "/reference/b/file"); + resolvedURi.Should().Be("https://www.elastic.co/docs/reference/elasticsearch/clients/net/b/file"); + + resolvedURi = uriResolver.Resolve(new Uri("elasticsearch://extend/c/file.md"), "/extend/c/file"); + resolvedURi.Should().Be("https://www.elastic.co/docs/extend/elasticsearch/c/file"); + } +} diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj b/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj new file mode 100644 index 000000000..8adb49ea6 --- /dev/null +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj @@ -0,0 +1,37 @@ + + + + + net9.0 + enable + enable + + false + true + Exe + + false + true + Documentation.Assembler.Tests + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file