Skip to content

Commit 2d82ed3

Browse files
retranretran
committed
Add AI agent discovery: llms.txt, llms-full.txt, per-page Markdown output
Generates three artifacts that make the docs consumable by AI agents and LLM-based tools: - llms.txt — full site index in llms.txt spec format: H1 title, blockquote summary, indented bullet links with descriptions in ToC order. Links point to index.md files. - llms-full.txt — complete Markdown content of every page in a single file, in ToC order, with page metadata (URL, Markdown permalink, description) and raw content per page. - {page}/index.md — clean Markdown version of every page (home, section, leaf). Internal links are rewritten from HTML paths (/path/to/page/) to Markdown paths (/path/to/page/index.md), including anchor fragments (/path/#section → /path/index.md#section). External links (https://) are left unchanged. - robots.txt — explicit Allow rules for 13 AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot, Bytespider, OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, Meta-ExternalAgent, Applebot-Extended, Diffbot), including training data collection bots, in production only. Coverage: 4257 index.md files match 4257 real HTML content pages (total minus 1105 alias redirects). llms.txt and llms-full.txt cover all non-draft pages (~4255).
1 parent d5ed1de commit 2d82ed3

10 files changed

Lines changed: 142 additions & 3 deletions

File tree

‎config/_default/hugo.toml‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,42 @@ replacements = "github.com/FortAwesome/Font-Awesome -> @fortawesome/fontawesome-
6363

6464
# Comment out if you don't want the "print entire section" link enabled.
6565
[outputs]
66-
section = ["HTML", "print"]
66+
home = ["HTML", "LLMSTXT", "LLMSFULL", "PAGEMD"]
67+
section = ["HTML", "print", "PAGEMD"]
6768
# MvM - add line below to print every page
68-
page = ["HTML", "print"]
69+
page = ["HTML", "print", "PAGEMD"]
6970

7071
# MvM - add line below to activate production of RSS pages
7172
[outputFormats]
7273
[outputFormats.RSS]
7374
mediatype = "application/rss"
7475
baseName = "rss"
7576

77+
# AI agent discovery: llms.txt index (home-level)
78+
[outputFormats.LLMSTXT]
79+
baseName = "llms"
80+
mediaType = "text/plain"
81+
isPlainText = true
82+
notAlternative = true
83+
84+
# AI agent discovery: llms-full.txt with full page content (home-level)
85+
[outputFormats.LLMSFULL]
86+
baseName = "llms-full"
87+
mediaType = "text/plain"
88+
isPlainText = true
89+
notAlternative = true
90+
91+
# AI agent discovery: clean Markdown version of each page at index.md
92+
[outputFormats.PAGEMD]
93+
baseName = "index"
94+
mediaType = "text/markdown"
95+
isPlainText = true
96+
notAlternative = true
97+
98+
[mediaTypes]
99+
[mediaTypes."text/markdown"]
100+
suffixes = ["md"]
101+
76102
# MvM - Set default priority for sitemap = 0.5
77103
# Can override in front matter for each section using
78104
# cascade:

‎config/production/hugo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ title = "Mendix Documentation"
88
# MvM - add line below to output MxDocsAlgolia at root of /public (master only)
99
[outputs]
1010
_merge = 'shallow'
11-
home = ["MxDocsAlgolia"]
11+
home = ["MxDocsAlgolia", "LLMSTXT", "LLMSFULL"]
1212

1313
# MvM - MxDocsAlgolia outputFormats parameters - to generate Algolia for Mendix
1414
[outputFormats]

‎layouts/_default/list.llmsfull.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# {{ .Site.Title }} — Full Documentation
2+
3+
> This file contains the complete Markdown content of all Mendix documentation pages for AI agent indexing.
4+
> For a page index with descriptions, see llms.txt.
5+
6+
{{- /* Walk the same tree as llms.txt: start from the landingpage root's children */ -}}
7+
{{- $docsRoot := index (where .Site.Sections "Type" "landingpage") 0 -}}
8+
9+
{{- /* Include the root (home/landing) page itself */ -}}
10+
{{- if $docsRoot -}}
11+
{{- printf "\n---\n\n# %s\n\nURL: %s\nMarkdown: %sindex.md" $docsRoot.Title $docsRoot.Permalink $docsRoot.Permalink -}}
12+
{{- with $docsRoot.Description -}}{{- printf "\nDescription: %s" . -}}{{- end -}}
13+
{{- printf "\n\n%s" ($docsRoot.RawContent | strings.ReplaceRE `\]\((/[^:)#" ]+/)(#[^)"]*)?\)` `](${1}index.md${2})`) -}}
14+
{{- range $docsRoot.Pages -}}
15+
{{- partial "llms-tree-full.txt" (dict "page" .) -}}
16+
{{- end -}}
17+
{{- end }}

‎layouts/_default/list.llmstxt.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# {{ .Site.Title }}
2+
3+
> Documentation for Mendix Studio Pro, platform features, APIs, and how-to guides.
4+
5+
{{- /* The docs root section has type: landingpage and url: /
6+
Its immediate children are the top-level sections (refguide, howto, etc.) */ -}}
7+
{{- $docsRoot := index (where .Site.Sections "Type" "landingpage") 0 -}}
8+
{{- if $docsRoot -}}
9+
{{- partial "llms-tree.txt" (dict "page" $docsRoot "depth" 0) -}}
10+
{{- end }}
11+
12+
## Optional
13+
14+
- [llms-full.txt]({{ .Site.BaseURL }}llms-full.txt): Full Markdown content of all documentation pages for offline indexing.

‎layouts/_default/list.pagemd.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# {{ .Title | strings.TrimSpace -}}
2+
3+
{{ $needSeparator := false -}}
4+
5+
{{/* Description */}}
6+
{{ with .Description | strings.TrimSpace }}
7+
8+
> {{ replace . "\n" "\n> " -}}
9+
{{ $needSeparator = true -}}
10+
{{ end -}}
11+
12+
{{/* Page content with index.md link rewriting */}}
13+
{{ $content := .RenderShortcodes | strings.TrimSpace | strings.ReplaceRE `\]\((/[^:)#" ]+/)(#[^)"]*)?\)` `](${1}index.md${2})` -}}
14+
{{ with $content -}}
15+
{{ if $needSeparator }}
16+
---
17+
18+
{{ else }}
19+
{{ end -}}
20+
{{ . }}
21+
{{ $needSeparator = true -}}
22+
{{ end -}}
23+
24+
{{/* Child pages list */}}
25+
{{ with .Pages -}}
26+
{{ if $needSeparator }}
27+
---
28+
29+
{{ else }}
30+
{{ end -}}
31+
Section pages:
32+
33+
{{ range . -}}
34+
- [{{ .Title | strings.TrimSpace }}]({{ .RelPermalink }}index.md)
35+
{{- with .Description | strings.TrimSpace }}: {{ . }}{{ end }}
36+
{{ end -}}
37+
{{ end -}}

‎layouts/_default/single.pagemd.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{{ .RawContent | strings.ReplaceRE `\]\((/[^:)#" ]+/)(#[^)"]*)?\)` `](${1}index.md${2})` }}

‎layouts/partials/llms-tree-full.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{{- $page := .page -}}
2+
{{- if and (not $page.Draft) (not $page.Params.private) -}}
3+
{{- printf "\n---\n\n# %s\n\nURL: %s\nMarkdown: %sindex.md" $page.Title $page.Permalink $page.Permalink -}}
4+
{{- with $page.Description -}}{{- printf "\nDescription: %s" . -}}{{- end -}}
5+
{{- printf "\n\n%s" ($page.RawContent | strings.ReplaceRE `\]\((/[^:)#" ]+/)(#[^)"]*)?\)` `](${1}index.md${2})`) -}}
6+
{{- if $page.IsSection -}}{{- range $page.Pages -}}{{- partial "llms-tree-full.txt" (dict "page" .) -}}{{- end -}}{{- end -}}
7+
{{- end -}}

‎layouts/partials/llms-tree.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{{- $page := .page -}}
2+
{{- $depth := .depth -}}
3+
{{- if and (not $page.Draft) (not $page.Params.private) -}}
4+
{{- $indent := strings.Repeat $depth " " -}}
5+
{{- $desc := "" -}}{{- with $page.Description -}}{{- $desc = printf ": %s" . -}}{{- end -}}
6+
{{- printf "\n%s- [%s](%sindex.md)%s" $indent $page.Title $page.Permalink $desc -}}
7+
{{- if $page.IsSection -}}{{- range $page.Pages -}}{{- partial "llms-tree.txt" (dict "page" . "depth" (add $depth 1)) -}}{{- end -}}{{- end -}}
8+
{{- end -}}

‎layouts/robots.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,28 @@ Disallow: /404.html
1717
Disallow: /
1818
{{- end }}
1919

20+
# AI crawlers — explicitly allow indexing and training data collection
21+
{{- $aiBots := slice
22+
"GPTBot" "OAI-SearchBot" "ChatGPT-User"
23+
"ClaudeBot" "Claude-SearchBot" "Claude-User"
24+
"Google-Extended"
25+
"Meta-ExternalAgent"
26+
"PerplexityBot"
27+
"Applebot-Extended"
28+
"Diffbot"
29+
"CCBot"
30+
"Bytespider" -}}
31+
{{- range $aiBots }}
32+
33+
User-agent: {{ . }}
34+
{{- if $isProduction }}
35+
Allow: /
36+
Disallow: /_includes/
37+
Disallow: /_print/
38+
Disallow: /attachments/
39+
{{- else }}
40+
Disallow: /
41+
{{- end }}
42+
{{- end }}
43+
2044
# End of robots.txt file

‎package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,10 @@
3131
"dependencies": {
3232
"@fortawesome/fontawesome-free": "^6.5.2",
3333
"bootstrap": "^5.3.3"
34+
},
35+
"allowScripts": {
36+
"github:google/docsy#01c827ea890e8e498f6046a7666a3031f318cc7f": true,
37+
"hugo-extended@0.156.0": true,
38+
"fsevents@2.3.3": true
3439
}
3540
}

0 commit comments

Comments
 (0)