diff --git a/README.md b/README.md index 46d7a6464..76d189a1e 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,9 @@ Options: --force Force a full rebuild of the destination folder (Default: null) Commands: - generate Converts a source markdown folder or file to an output folder - serve Continuously serve a documentation folder at http://localhost:3000. + generate Converts a source markdown folder or file to an output folder + serve Continuously serve a documentation folder at http://localhost:3000. + diff validate Validates redirect rules have been applied to the current branch. File systems changes will be reflected without having to restart the server. ``` @@ -118,6 +119,16 @@ https://github.com/elastic/{your-repository}/settings/pages --- +## Validating redirection rules + +If documentation is moved, renamed or deleted, `docs-builder` can verify if changes in the working branch in relation to the default branch are reflected in the repository's `redirects.yml`. Verification in the local machine is currently supported. + +`docs-builder diff validate ` + +`` is an optional parameter to customize the documentation folder path. It defaults to `docs`. + +--- + ## Run without docker You can use the .NET CLI to publish a self-contained `docs-builder` native code @@ -140,7 +151,7 @@ existing surveyed tools # Local Development -## Preqrequisites +## Prerequisites - [.NET 9.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/9.0) - [Node.js 22.13.1 (LTS)](https://nodejs.org/en/blog/release/v22.13.1) diff --git a/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj b/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj index 52cc0b08e..5a53168d4 100644 --- a/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj +++ b/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj @@ -13,6 +13,7 @@ + diff --git a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs new file mode 100644 index 000000000..363b60987 --- /dev/null +++ b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs @@ -0,0 +1,122 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Documentation.Diagnostics; +using ProcNet; +using ProcNet.Std; + +namespace Elastic.Documentation.Tooling.ExternalCommands; + +public abstract class ExternalCommandExecutor(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) +{ + protected IDirectoryInfo WorkingDirectory => workingDirectory; + protected void ExecIn(Dictionary environmentVars, string binary, params string[] args) + { + var arguments = new ExecArguments(binary, args) + { + WorkingDirectory = workingDirectory.FullName, + Environment = environmentVars + }; + var result = Proc.Exec(arguments); + if (result != 0) + collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); + } + + protected void ExecInSilent(Dictionary environmentVars, string binary, params string[] args) + { + var arguments = new StartArguments(binary, args) + { + Environment = environmentVars, + WorkingDirectory = workingDirectory.FullName, + ConsoleOutWriter = NoopConsoleWriter.Instance + }; + var result = Proc.Start(arguments); + if (result.ExitCode != 0) + collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); + } + + protected string[] CaptureMultiple(string binary, params string[] args) + { + // Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try + Exception? e = null; + for (var i = 0; i <= 9; i++) + { + try + { + return CaptureOutput(); + } + catch (Exception ex) + { + if (ex is not null) + e = ex; + } + } + + if (e is not null) + collector.EmitError("", "failure capturing stdout", e); + + return []; + + string[] CaptureOutput() + { + var arguments = new StartArguments(binary, args) + { + WorkingDirectory = workingDirectory.FullName, + Timeout = TimeSpan.FromSeconds(3), + WaitForExit = TimeSpan.FromSeconds(3), + ConsoleOutWriter = NoopConsoleWriter.Instance + }; + var result = Proc.Start(arguments); + var output = result.ExitCode != 0 + ? throw new Exception($"Exit code is not 0. Received {result.ExitCode} from {binary}: {workingDirectory}") + : result.ConsoleOut.Select(x => x.Line).ToArray() ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"); + return output; + } + } + + + protected string Capture(string binary, params string[] args) => Capture(false, binary, args); + + protected string Capture(bool muteExceptions, string binary, params string[] args) + { + // Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try + Exception? e = null; + for (var i = 0; i <= 9; i++) + { + try + { + return CaptureOutput(); + } + catch (Exception ex) + { + if (ex is not null) + e = ex; + } + } + + if (e is not null && !muteExceptions) + collector.EmitError("", "failure capturing stdout", e); + + return string.Empty; + + string CaptureOutput() + { + var arguments = new StartArguments(binary, args) + { + WorkingDirectory = workingDirectory.FullName, + Timeout = TimeSpan.FromSeconds(3), + WaitForExit = TimeSpan.FromSeconds(3), + ConsoleOutWriter = NoopConsoleWriter.Instance + }; + var result = Proc.Start(arguments); + var line = (result.ExitCode, muteExceptions) switch + { + (0, _) or (not 0, true) => result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"), + (not 0, false) => throw new Exception($"Exit code is not 0. Received {result.ExitCode} from {binary}: {workingDirectory}") + }; + return line; + } + } +} diff --git a/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/NoopConsoleWriter.cs b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/NoopConsoleWriter.cs new file mode 100644 index 000000000..19738a478 --- /dev/null +++ b/src/tooling/Elastic.Documentation.Tooling/ExternalCommands/NoopConsoleWriter.cs @@ -0,0 +1,16 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using ProcNet.Std; + +namespace Elastic.Documentation.Tooling.ExternalCommands; + +public class NoopConsoleWriter : IConsoleOutWriter +{ + public static readonly NoopConsoleWriter Instance = new(); + + public void Write(Exception e) { } + + public void Write(ConsoleOut consoleOut) { } +} diff --git a/src/tooling/docs-assembler/Sourcing/GitFacade.cs b/src/tooling/docs-assembler/Sourcing/GitFacade.cs index f64a38888..6a518c3db 100644 --- a/src/tooling/docs-assembler/Sourcing/GitFacade.cs +++ b/src/tooling/docs-assembler/Sourcing/GitFacade.cs @@ -4,7 +4,7 @@ using System.IO.Abstractions; using Elastic.Documentation.Diagnostics; -using ProcNet; +using Elastic.Documentation.Tooling.ExternalCommands; namespace Documentation.Assembler.Sourcing; @@ -24,73 +24,25 @@ public interface IGitRepository // This git repository implementation is optimized for pull and fetching single commits. // It uses `git pull --depth 1` and `git fetch --depth 1` to minimize the amount of data transferred. -public class SingleCommitOptimizedGitRepository(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) : IGitRepository +public class SingleCommitOptimizedGitRepository(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) : ExternalCommandExecutor(collector, workingDirectory), IGitRepository { - public string GetCurrentCommit() => Capture("git", "rev-parse", "HEAD"); - - public void Init() => ExecIn("git", "init"); - public bool IsInitialized() => Directory.Exists(Path.Combine(workingDirectory.FullName, ".git")); - public void Pull(string branch) => ExecIn("git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff", "origin", branch); - public void Fetch(string reference) => ExecIn("git", "fetch", "--no-tags", "--prune", "--no-recurse-submodules", "--depth", "1", "origin", reference); - public void EnableSparseCheckout(string folder) => ExecIn("git", "sparse-checkout", "set", folder); - public void DisableSparseCheckout() => ExecIn("git", "sparse-checkout", "disable"); - public void Checkout(string reference) => ExecIn("git", "checkout", "--force", reference); - - public void GitAddOrigin(string origin) => ExecIn("git", "remote", "add", "origin", origin); - - private void ExecIn(string binary, params string[] args) - { - var arguments = new ExecArguments(binary, args) - { - WorkingDirectory = workingDirectory.FullName, - Environment = new Dictionary - { - // Disable git editor prompts: - // There are cases where `git pull` would prompt for an editor to write a commit message. - // This env variable prevents that. - { "GIT_EDITOR", "true" } - }, - }; - var result = Proc.Exec(arguments); - if (result != 0) - collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); - } - private string Capture(string binary, params string[] args) + private static readonly Dictionary EnvironmentVars = new() { - // Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try - Exception? e = null; - for (var i = 0; i <= 9; i++) - { - try - { - return CaptureOutput(); - } - catch (Exception ex) - { - if (ex is not null) - e = ex; - } - } + // Disable git editor prompts: + // There are cases where `git pull` would prompt for an editor to write a commit message. + // This env variable prevents that. + { "GIT_EDITOR", "true" } + }; - if (e is not null) - collector.EmitError("", "failure capturing stdout", e); + public string GetCurrentCommit() => Capture("git", "rev-parse", "HEAD"); - return string.Empty; + public void Init() => ExecIn(EnvironmentVars, "git", "init"); + public bool IsInitialized() => Directory.Exists(Path.Combine(WorkingDirectory.FullName, ".git")); + public void Pull(string branch) => ExecIn(EnvironmentVars, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff", "origin", branch); + public void Fetch(string reference) => ExecIn(EnvironmentVars, "git", "fetch", "--no-tags", "--prune", "--no-recurse-submodules", "--depth", "1", "origin", reference); + public void EnableSparseCheckout(string folder) => ExecIn(EnvironmentVars, "git", "sparse-checkout", "set", folder); + public void DisableSparseCheckout() => ExecIn(EnvironmentVars, "git", "sparse-checkout", "disable"); + public void Checkout(string reference) => ExecIn(EnvironmentVars, "git", "checkout", "--force", reference); - string CaptureOutput() - { - var arguments = new StartArguments(binary, args) - { - WorkingDirectory = workingDirectory.FullName, - Timeout = TimeSpan.FromSeconds(3), - WaitForExit = TimeSpan.FromSeconds(3), - ConsoleOutWriter = NoopConsoleWriter.Instance - }; - var result = Proc.Start(arguments); - var line = result.ExitCode != 0 - ? throw new Exception($"Exit code is not 0. Received {result.ExitCode} from {binary}: {workingDirectory}") - : result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"); - return line; - } - } + public void GitAddOrigin(string origin) => ExecIn(EnvironmentVars, "git", "remote", "add", "origin", origin); } diff --git a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index dd822b2d7..33d4aa322 100644 --- a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -237,15 +237,6 @@ private static void FetchAndCheckout(IGitRepository git, Repository repository, } } -public class NoopConsoleWriter : IConsoleOutWriter -{ - public static readonly NoopConsoleWriter Instance = new(); - - public void Write(Exception e) { } - - public void Write(ConsoleOut consoleOut) { } -} - public record CheckoutResult { public static string LinkRegistrySnapshotFileName => "link-index.snapshot.json"; diff --git a/src/tooling/docs-builder/Cli/DiffCommands.cs b/src/tooling/docs-builder/Cli/DiffCommands.cs new file mode 100644 index 000000000..8e5691aef --- /dev/null +++ b/src/tooling/docs-builder/Cli/DiffCommands.cs @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions; +using Actions.Core.Services; +using ConsoleAppFramework; +using Documentation.Builder.Tracking; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Tooling.Diagnostics.Console; +using Elastic.Documentation.Tooling.Filters; +using Microsoft.Extensions.Logging; + +namespace Documentation.Builder.Cli; + +internal sealed class DiffCommands(ILoggerFactory logger, ICoreService githubActionsService) +{ + /// + /// Validates redirect updates in the current branch using the redirects file against changes reported by git. + /// + /// The baseline path to perform the check + /// + [SuppressMessage("Usage", "CA2254:Template should be a static expression")] + [Command("validate")] + [ConsoleAppFilter] + [ConsoleAppFilter] + public async Task ValidateRedirects([Argument] string? path = null, Cancel ctx = default) + { + var log = logger.CreateLogger(); + ConsoleApp.Log = msg => log.LogInformation(msg); + ConsoleApp.LogError = msg => log.LogError(msg); + + path ??= "docs"; + + await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService).StartAsync(ctx); + + var fs = new FileSystem(); + var root = fs.DirectoryInfo.New(Paths.WorkingDirectoryRoot.FullName); + + var buildContext = new BuildContext(collector, fs, fs, root.FullName, null); + var sourceFile = buildContext.ConfigurationPath; + var redirectFileName = sourceFile.Name.StartsWith('_') ? "_redirects.yml" : "redirects.yml"; + var redirectFileInfo = sourceFile.FileSystem.FileInfo.New(Path.Combine(sourceFile.Directory!.FullName, redirectFileName)); + + var redirectFileParser = new RedirectFile(redirectFileInfo, buildContext); + var redirects = redirectFileParser.Redirects; + + if (redirects is null) + { + collector.EmitError(redirectFileInfo, "It was not possible to parse the redirects file."); + await collector.StopAsync(ctx); + return collector.Errors; + } + + var tracker = new LocalGitRepositoryTracker(collector, root); + var changed = tracker.GetChangedFiles(path); + + foreach (var notFound in changed.DistinctBy(c => c.FilePath).Where(c => c.ChangeType is GitChangeType.Deleted or GitChangeType.Renamed + && !redirects.ContainsKey(c is RenamedGitChange renamed ? renamed.OldFilePath : c.FilePath))) + { + if (notFound is RenamedGitChange renamed) + { + collector.EmitError(redirectFileInfo.Name, + $"File '{renamed.OldFilePath}' was renamed to '{renamed.NewFilePath}' but it has no redirect configuration set."); + } + else if (notFound.ChangeType is GitChangeType.Deleted) + { + collector.EmitError(redirectFileInfo.Name, + $"File '{notFound.FilePath}' was deleted but it has no redirect targets. This will lead to broken links."); + } + } + + await collector.StopAsync(ctx); + return collector.Errors; + } +} diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 7baebbd2e..15a0bbfb3 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -17,5 +17,6 @@ var app = ConsoleApp.Create(); app.Add(); app.Add("inbound-links"); +app.Add("diff"); await app.RunAsync(args).ConfigureAwait(false); diff --git a/src/tooling/docs-builder/Tracking/IRepositoryTracker.cs b/src/tooling/docs-builder/Tracking/IRepositoryTracker.cs new file mode 100644 index 000000000..de01a1f01 --- /dev/null +++ b/src/tooling/docs-builder/Tracking/IRepositoryTracker.cs @@ -0,0 +1,20 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Documentation.Builder.Tracking; + +public enum GitChangeType +{ + Added, + Modified, + Deleted, + Renamed, + Untracked, + Other +} + +public interface IRepositoryTracker +{ + IEnumerable GetChangedFiles(string lookupPath); +} diff --git a/src/tooling/docs-builder/Tracking/LocalGitRepositoryTracker.cs b/src/tooling/docs-builder/Tracking/LocalGitRepositoryTracker.cs new file mode 100644 index 000000000..f1280d13b --- /dev/null +++ b/src/tooling/docs-builder/Tracking/LocalGitRepositoryTracker.cs @@ -0,0 +1,85 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Tooling.ExternalCommands; + +namespace Documentation.Builder.Tracking; + +public record GitChange(string FilePath, GitChangeType ChangeType); +public record RenamedGitChange(string OldFilePath, string NewFilePath, GitChangeType ChangeType) : GitChange(OldFilePath, ChangeType); + +public class LocalGitRepositoryTracker(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) : ExternalCommandExecutor(collector, workingDirectory), IRepositoryTracker +{ + public IEnumerable GetChangedFiles(string lookupPath) + { + var defaultBranch = GetDefaultBranch(); + var commitChanges = CaptureMultiple("git", "diff", "--name-status", $"{defaultBranch}...HEAD", "--", $"./{lookupPath}"); + var localChanges = CaptureMultiple("git", "status", "--porcelain"); + ExecInSilent([], "git", "stash", "push", "--", $"./{lookupPath}"); + var localUnstagedChanges = CaptureMultiple("git", "stash", "show", "--name-status", "-u"); + ExecInSilent([], "git", "stash", "pop"); + + return [.. GetCommitChanges(commitChanges), .. GetLocalChanges(localChanges), .. GetCommitChanges(localUnstagedChanges)]; + } + + private string GetDefaultBranch() + { + if (!Capture(true, "git", "merge-base", "-a", "HEAD", "main").StartsWith("fatal", StringComparison.InvariantCulture)) + return "main"; + if (!Capture(true, "git", "merge-base", "-a", "HEAD", "master").StartsWith("fatal", StringComparison.InvariantCulture)) + return "master"; + return Capture("git", "symbolic-ref", "refs/remotes/origin/HEAD").Split('/').Last(); + } + + private static IEnumerable GetCommitChanges(string[] changes) + { + foreach (var change in changes) + { + var parts = change.AsSpan().TrimStart(); + if (parts.Length < 2) + continue; + + var changeType = parts[0] switch + { + 'A' => GitChangeType.Added, + 'M' => GitChangeType.Modified, + 'D' => GitChangeType.Deleted, + 'R' => GitChangeType.Renamed, + _ => GitChangeType.Other + }; + + yield return new GitChange(change.Split('\t')[1], changeType); + } + } + + private static IEnumerable GetLocalChanges(string[] changes) + { + foreach (var change in changes) + { + var changeStatusCode = change.AsSpan(); + if (changeStatusCode.Length < 2) + continue; + + var changeType = (changeStatusCode[0], changeStatusCode[1]) switch + { + ('R', _) or (_, 'R') => GitChangeType.Renamed, + ('D', _) or (_, 'D') when changeStatusCode[0] != 'A' => GitChangeType.Deleted, + ('?', '?') => GitChangeType.Untracked, + ('A', _) or (_, 'A') => GitChangeType.Added, + ('M', _) or (_, 'M') => GitChangeType.Modified, + _ => GitChangeType.Other + }; + + var changeParts = change.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + yield return changeType switch + { + GitChangeType.Renamed => new RenamedGitChange(changeParts[1], changeParts[3], changeType), + _ => new GitChange(changeParts[1], changeType) + }; + } + } +}