From 858ab57c8d2558770b5a0cc888799c5b79b7b0fb Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 18 Mar 2026 17:52:18 +0100 Subject: [PATCH 1/5] feat: add agent-friendly docs (llms.txt, markdown endpoints, agent signaling) Implement the Agent-Friendly Documentation spec (agentdocsspec.com) to make docs discoverable and consumable by AI agents. --- .github/workflows/docs.yml | 27 +++ docs-site/astro.config.mjs | 6 + docs-site/package-lock.json | 109 +++++++++++ docs-site/package.json | 1 + docs-site/plugins/astro-agent-docs.mjs | 190 +++++++++++++++++++ docs-site/plugins/rehype-agent-signaling.mjs | 38 ++++ docs-site/src/styles/elements.css | 16 ++ 7 files changed, 387 insertions(+) create mode 100644 docs-site/plugins/astro-agent-docs.mjs create mode 100644 docs-site/plugins/rehype-agent-signaling.mjs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7b082f163..2c3482f17 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -214,6 +214,33 @@ jobs: publish_branch: docs-deployment destination_dir: ${{ env.DOCS_VERSION }} + - name: Check if this is the latest version + id: check-latest + run: | + LATEST=$(jq -r '.versions[] | select(.latest == true) | .version' docs-site/versions.json) + if [[ "$LATEST" == "$DOCS_VERSION" ]]; then + echo "is_latest=true" >> "$GITHUB_OUTPUT" + echo "✅ Version ${DOCS_VERSION} is latest — will deploy llms.txt to root" + else + echo "is_latest=false" >> "$GITHUB_OUTPUT" + echo "ℹ️ Version ${DOCS_VERSION} is not latest (${LATEST}) — skipping root llms.txt" + fi + + - name: Deploy llms.txt to root + if: steps.check-latest.outputs.is_latest == 'true' + run: | + mkdir -p root-llms + cp docs-site/dist/llms.txt root-llms/llms.txt + + - name: Publish root llms.txt to docs-deployment branch + if: steps.check-latest.outputs.is_latest == 'true' + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./root-llms + publish_branch: docs-deployment + keep_files: true + # Deploy docs-deployment branch to IC asset canister. # Runs when at least one publish job succeeded (skipped on PR builds). # always() prevents auto-skip when publish-versioned-docs is skipped (e.g. on main branch pushes). diff --git a/docs-site/astro.config.mjs b/docs-site/astro.config.mjs index 8646b8648..fe7b8b28d 100644 --- a/docs-site/astro.config.mjs +++ b/docs-site/astro.config.mjs @@ -2,6 +2,8 @@ import { defineConfig } from 'astro/config'; import starlight from '@astrojs/starlight'; import rehypeExternalLinks from 'rehype-external-links'; import rehypeRewriteLinks from './plugins/rehype-rewrite-links.mjs'; +import rehypeAgentSignaling from './plugins/rehype-agent-signaling.mjs'; +import agentDocs from './plugins/astro-agent-docs.mjs'; // https://astro.build/config export default defineConfig({ @@ -15,9 +17,13 @@ export default defineConfig({ rehypeRewriteLinks, // Open external links in new tab [rehypeExternalLinks, { target: '_blank', rel: ['noopener', 'noreferrer'] }], + // Inject hidden llms.txt directive for agent discovery + rehypeAgentSignaling, ], }, integrations: [ + // Generate .md endpoints and llms.txt for agent-friendly docs + agentDocs(), starlight({ title: 'ICP CLI', description: 'Command-line tool for developing and deploying applications on the Internet Computer Protocol (ICP)', diff --git a/docs-site/package-lock.json b/docs-site/package-lock.json index d5606d881..5a06c7f20 100644 --- a/docs-site/package-lock.json +++ b/docs-site/package-lock.json @@ -11,6 +11,7 @@ "@astrojs/check": "^0.9.4", "@astrojs/starlight": "^0.37.3", "astro": "^5.6.1", + "gray-matter": "^4.0.3", "sharp": "^0.33.5" }, "devDependencies": { @@ -3567,6 +3568,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/estree-util-attach-comments": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-3.0.0.tgz", @@ -3682,6 +3696,18 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -3792,6 +3818,43 @@ "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", "license": "ISC" }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/h3": { "version": "1.15.5", "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.5.tgz", @@ -4322,6 +4385,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -4410,6 +4482,15 @@ "integrity": "sha512-H8jvkz1O50L3dMZCsLqiuB2tA7muqbSg1AtGEkN0leAqGjsUzDJir3Zwr02BhqdcITPg3ei3mZ+HjMocAknhhg==", "license": "MIT" }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", @@ -6441,6 +6522,19 @@ "node": ">=11.0.0" } }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", @@ -6588,6 +6682,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/stream-replace-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/stream-replace-string/-/stream-replace-string-2.0.0.tgz", @@ -6640,6 +6740,15 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/style-to-js": { "version": "1.1.21", "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", diff --git a/docs-site/package.json b/docs-site/package.json index 974db571b..d0a83b637 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -13,6 +13,7 @@ "@astrojs/check": "^0.9.4", "@astrojs/starlight": "^0.37.3", "astro": "^5.6.1", + "gray-matter": "^4.0.3", "sharp": "^0.33.5" }, "devDependencies": { diff --git a/docs-site/plugins/astro-agent-docs.mjs b/docs-site/plugins/astro-agent-docs.mjs new file mode 100644 index 000000000..ee1e21dea --- /dev/null +++ b/docs-site/plugins/astro-agent-docs.mjs @@ -0,0 +1,190 @@ +/** + * Astro integration for Agent-Friendly Documentation. + * Implements https://agentdocsspec.com: + * + * 1. Markdown endpoints — serves a clean .md file alongside every HTML page + * 2. llms.txt — discovery index listing all pages with links to .md endpoints + * + * Runs in the astro:build:done hook so it operates on the final build output. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import matter from "gray-matter"; + +// Files excluded from the docs site (must match content.config.ts glob exclusions). +const EXCLUDED = ["schemas/**", "**/README.md", "VERSIONED_DOCS.md"]; + +function isExcluded(file) { + if (file === "VERSIONED_DOCS.md" || file.endsWith("/README.md") || file === "README.md") { + return true; + } + if (file.startsWith("schemas/")) return true; + return false; +} + +/** + * Sidebar section definitions — mirrors the sidebar in astro.config.mjs. + * Each entry maps a directory prefix to a section label for llms.txt grouping. + * Order matters: entries are checked in order, longest prefix match wins. + */ +const SECTIONS = [ + { dir: "guides", label: "Guides" }, + { dir: "concepts", label: "Concepts" }, + { dir: "reference", label: "Reference" }, + { dir: "migration", label: "Other" }, +]; + +// UTF-8 BOM — ensures browsers interpret .md files correctly even without +// a charset=utf-8 in the Content-Type header. +const BOM = "\uFEFF"; + +/** Strip YAML frontmatter and HTML comments, prepend title heading. */ +function cleanMarkdown(raw) { + const { data, content } = matter(raw); + const body = content.replace(//g, "").trim(); + const title = data.title ? `# ${data.title}\n\n` : ""; + return BOM + title + body + "\n"; +} + +/** Find the best matching section for a file path (longest prefix wins). */ +function findSection(filePath) { + let best = null; + for (const section of SECTIONS) { + if ( + filePath.startsWith(section.dir + "/") && + (!best || section.dir.length > best.dir.length) + ) { + best = section; + } + } + return best; +} + +/** Generate llms.txt content from collected page metadata. */ +function generateLlmsTxt(pages, siteUrl, basePath) { + const base = (siteUrl + basePath).replace(/\/$/, ""); + + const lines = [ + "# ICP CLI Documentation", + "", + "> Command-line tool for developing and deploying applications on the Internet Computer Protocol (ICP).", + "", + ]; + + // Root index page + const rootIndex = pages.find((p) => p.file === "index.md"); + if (rootIndex) { + lines.push( + `- [${rootIndex.title}](${base}/index.md): ${rootIndex.description}` + ); + lines.push(""); + } + + // Ungrouped top-level pages (not index, not in any section directory) + const topLevel = pages.filter( + (p) => p.file !== "index.md" && !findSection(p.file) + ); + if (topLevel.length > 0) { + lines.push("## Start Here"); + lines.push(""); + for (const page of topLevel) { + const url = `${base}/${page.file}`; + const entry = page.description + ? `- [${page.title}](${url}): ${page.description}` + : `- [${page.title}](${url})`; + lines.push(entry); + } + lines.push(""); + } + + // Group pages by section + const grouped = new Map(); + for (const section of SECTIONS) { + grouped.set(section.label, []); + } + + for (const page of pages) { + if (page.file === "index.md") continue; + const section = findSection(page.file); + if (section) { + grouped.get(section.label).push(page); + } + } + + // Emit sections + for (const [label, sectionPages] of grouped) { + if (sectionPages.length === 0) continue; + + sectionPages.sort((a, b) => a.order - b.order); + + lines.push(`## ${label}`); + lines.push(""); + for (const page of sectionPages) { + const url = `${base}/${page.file}`; + const entry = page.description + ? `- [${page.title}](${url}): ${page.description}` + : `- [${page.title}](${url})`; + lines.push(entry); + } + lines.push(""); + } + + return lines.join("\n"); +} + +export default function agentDocs() { + let siteUrl = ""; + let basePath = "/"; + + return { + name: "agent-docs", + hooks: { + "astro:config:done": ({ config }) => { + siteUrl = (config.site || "").replace(/\/$/, ""); + basePath = config.base || "/"; + if (!basePath.endsWith("/")) basePath += "/"; + }, + "astro:build:done": async ({ dir, logger }) => { + const outDir = fileURLToPath(dir); + // Docs source lives at ../docs relative to the docs-site directory. + const docsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../docs"); + + const files = fs + .globSync("**/*.md", { cwd: docsDir }) + .filter((f) => !isExcluded(f)); + const pages = []; + + // 1. Generate markdown endpoints + for (const file of files) { + if (isExcluded(file)) continue; + + const raw = fs.readFileSync(path.join(docsDir, file), "utf-8"); + const { data: frontmatter } = matter(raw); + + // Write cleaned .md to build output + const outFile = path.join(outDir, file); + fs.mkdirSync(path.dirname(outFile), { recursive: true }); + fs.writeFileSync(outFile, cleanMarkdown(raw)); + + pages.push({ + file, + title: frontmatter.title || path.basename(file, ".md"), + description: frontmatter.description || "", + order: frontmatter.sidebar?.order ?? 999, + }); + } + + logger.info(`Generated ${pages.length} markdown endpoints`); + + // 2. Generate llms.txt + const llmsTxt = generateLlmsTxt(pages, siteUrl, basePath); + fs.writeFileSync(path.join(outDir, "llms.txt"), llmsTxt); + logger.info( + `Generated llms.txt (${llmsTxt.length} chars, ${pages.length} pages)` + ); + }, + }, + }; +} diff --git a/docs-site/plugins/rehype-agent-signaling.mjs b/docs-site/plugins/rehype-agent-signaling.mjs new file mode 100644 index 000000000..b2d7dbfb3 --- /dev/null +++ b/docs-site/plugins/rehype-agent-signaling.mjs @@ -0,0 +1,38 @@ +/** + * Rehype plugin that injects an agent signaling directive at the top of every page. + * Part of the Agent-Friendly Documentation spec (https://agentdocsspec.com). + * + * Adds a visually-hidden blockquote pointing agents to /llms.txt. + * Uses CSS clip-rect (not display:none) so it survives HTML-to-markdown conversion. + */ + +export default function rehypeAgentSignaling() { + return (tree) => { + const blockquote = { + type: "element", + tagName: "blockquote", + properties: { className: ["agent-signaling"] }, + children: [ + { + type: "element", + tagName: "p", + properties: {}, + children: [ + { + type: "text", + value: "For AI agents: Documentation index at ", + }, + { + type: "element", + tagName: "a", + properties: { href: "/llms.txt" }, + children: [{ type: "text", value: "/llms.txt" }], + }, + ], + }, + ], + }; + + tree.children.unshift(blockquote); + }; +} diff --git a/docs-site/src/styles/elements.css b/docs-site/src/styles/elements.css index 6e91431af..09fba6b61 100644 --- a/docs-site/src/styles/elements.css +++ b/docs-site/src/styles/elements.css @@ -1,4 +1,20 @@ @layer elements { + /** + * Agent signaling — visually hidden but survives HTML-to-markdown conversion. + * Uses clip-rect (not display:none) so agents parsing HTML can discover /llms.txt. + * See: https://agentdocsspec.com + */ + .agent-signaling { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; + } /** * Common */ From b6c2f84211b93245e10889f1da749bd7b017de71 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 18 Mar 2026 17:58:35 +0100 Subject: [PATCH 2/5] fix: use Node 22 in docs workflow for fs.globSync support --- .github/workflows/docs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2c3482f17..eb5c02043 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,7 +42,7 @@ jobs: - name: Setup Node uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v4.2.0 with: - node-version: '20' + node-version: '22' cache: 'npm' cache-dependency-path: docs-site/package-lock.json @@ -69,7 +69,7 @@ jobs: - name: Setup Node uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v4.2.0 with: - node-version: '20' + node-version: '22' - name: Prepare root files run: | @@ -129,7 +129,7 @@ jobs: - name: Setup Node uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v4.2.0 with: - node-version: '20' + node-version: '22' cache: 'npm' cache-dependency-path: docs-site/package-lock.json @@ -168,7 +168,7 @@ jobs: - name: Setup Node uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v4.2.0 with: - node-version: '20' + node-version: '22' cache: 'npm' cache-dependency-path: docs-site/package-lock.json From 25bde4d76975a8ee4d4750469a7a20c4f9701642 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Thu, 19 Mar 2026 08:07:01 +0100 Subject: [PATCH 3/5] fix: move root llms.txt deployment to publish-root-files Root llms.txt is now copied from the latest version's folder on the docs-deployment branch during publish-root-files, instead of being deployed by publish-versioned-docs. This ensures root llms.txt stays in sync when versions.json is bumped (which only triggers publish-root-files, not publish-versioned-docs). --- .github/workflows/docs.yml | 42 ++++++++++++++------------------------ 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eb5c02043..07a1bf935 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -109,6 +109,21 @@ jobs: EOF + echo "LATEST_VERSION=${LATEST_VERSION}" >> $GITHUB_ENV + + - name: Copy llms.txt from latest version on docs-deployment branch + run: | + # Fetch the llms.txt from the latest version's folder on docs-deployment. + # This file was deployed by publish-versioned-docs and is always in sync + # with the versioned .md endpoints it links to. + git fetch origin docs-deployment --depth=1 + if git show "origin/docs-deployment:${LATEST_VERSION}/llms.txt" > root/llms.txt 2>/dev/null; then + echo "✅ Copied llms.txt from /${LATEST_VERSION}/llms.txt to root" + else + echo "⚠️ No llms.txt found at /${LATEST_VERSION}/llms.txt on docs-deployment — skipping" + rm -f root/llms.txt + fi + - name: Deploy root files to docs-deployment branch uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: @@ -214,33 +229,6 @@ jobs: publish_branch: docs-deployment destination_dir: ${{ env.DOCS_VERSION }} - - name: Check if this is the latest version - id: check-latest - run: | - LATEST=$(jq -r '.versions[] | select(.latest == true) | .version' docs-site/versions.json) - if [[ "$LATEST" == "$DOCS_VERSION" ]]; then - echo "is_latest=true" >> "$GITHUB_OUTPUT" - echo "✅ Version ${DOCS_VERSION} is latest — will deploy llms.txt to root" - else - echo "is_latest=false" >> "$GITHUB_OUTPUT" - echo "ℹ️ Version ${DOCS_VERSION} is not latest (${LATEST}) — skipping root llms.txt" - fi - - - name: Deploy llms.txt to root - if: steps.check-latest.outputs.is_latest == 'true' - run: | - mkdir -p root-llms - cp docs-site/dist/llms.txt root-llms/llms.txt - - - name: Publish root llms.txt to docs-deployment branch - if: steps.check-latest.outputs.is_latest == 'true' - uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./root-llms - publish_branch: docs-deployment - keep_files: true - # Deploy docs-deployment branch to IC asset canister. # Runs when at least one publish job succeeded (skipped on PR builds). # always() prevents auto-skip when publish-versioned-docs is skipped (e.g. on main branch pushes). From 0c0bb794f2a5ca61d471fc31106d713b4aaf5898 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Thu, 19 Mar 2026 08:13:52 +0100 Subject: [PATCH 4/5] feat: add version navigation to root llms.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepend version links to the root llms.txt during publish-root-files. Only the root copy gets version navigation — versioned copies stay static to avoid stale cross-references when new versions are released. --- .github/workflows/docs.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 07a1bf935..26e5e2d69 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -124,6 +124,26 @@ jobs: rm -f root/llms.txt fi + - name: Prepend version navigation to root llms.txt + if: hashFiles('root/llms.txt') != '' + run: | + # Build a version navigation header from versions.json. + # Only the root llms.txt gets this — versioned copies stay static. + SITE="${PUBLIC_SITE}" + HEADER="This documentation is for v${LATEST_VERSION}. Other versions are available:" + + VERSION_LINES="" + for version in $(jq -r '.versions[].version' docs-site/versions.json); do + if [[ "$version" != "$LATEST_VERSION" ]]; then + VERSION_LINES="${VERSION_LINES}\n- [v${version}](${SITE}/${version}/llms.txt)" + fi + done + VERSION_LINES="${VERSION_LINES}\n- [Development (main)](${SITE}/main/llms.txt)" + + # Insert after the description blockquote (line starting with ">") + sed -i "s|^> .*|&\n\n${HEADER}${VERSION_LINES}|" root/llms.txt + echo "✅ Added version navigation to root llms.txt" + - name: Deploy root files to docs-deployment branch uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: From 577d00b23bc8abda1ae2ee2ce6bd3eb4c42ddb80 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Thu, 19 Mar 2026 08:17:00 +0100 Subject: [PATCH 5/5] docs: update VERSIONED_DOCS.md for agent-friendly docs and remove beta section - Add agent-friendly docs section explaining llms.txt and .md endpoints - Update deployment branch structure to show llms.txt and .md files - Update workflow trigger table to mention llms.txt handling - Remove outdated beta versions section (pre-release tags/branches are now excluded from the docs workflow) --- docs/VERSIONED_DOCS.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/VERSIONED_DOCS.md b/docs/VERSIONED_DOCS.md index e3ef92535..f916b12fe 100644 --- a/docs/VERSIONED_DOCS.md +++ b/docs/VERSIONED_DOCS.md @@ -22,14 +22,19 @@ All built assets live on the `docs-deployment` branch: ``` ├── index.html # Redirects to latest version (or /main/ if no releases) +├── llms.txt # Agent-friendly docs index (copied from latest version) ├── versions.json # List of available versions ├── icp.yaml # IC asset canister config ├── .ic-assets.json5 # Asset routing/headers config ├── .icp/data/mappings/ic.ids.json # Canister ID mapping ├── .well-known/ic-domains # Custom domain verification ├── main/ # Main branch docs (always updated) +│ ├── llms.txt # Agent docs index for this version +│ ├── quickstart.md # Markdown endpoints (clean, no frontmatter) +│ ├── guides/*.md +│ └── ... # HTML pages (Starlight output) ├── 0.1/ # Version 0.1 docs -├── 0.2/ # Version 0.2 docs +├── 0.2/ # Version 0.2 docs (same structure as main/) └── ... ``` @@ -39,9 +44,11 @@ All built assets live on the `docs-deployment` branch: | Trigger | Action | |---------|--------| -| Tag `v*` (e.g., `v0.2.0`) | Deploys to `/0.2/` (major.minor only) | +| Tag `v*` (e.g., `v0.2.0`) | Deploys to `/0.2/` with HTML, `.md` endpoints, and `llms.txt` | | Branch `docs/v*` (e.g., `docs/v0.1`) | Updates `/0.1/` (for cherry-picks / fixes to old versions) | -| Push to `main` | Deploys to `/main/`, updates root `index.html` and `versions.json` | +| Push to `main` | Deploys to `/main/`, updates root `index.html`, `versions.json`, and root `llms.txt` | + +Pre-release tags (e.g., `v0.2.0-beta.0`) and pre-release doc branches (e.g., `docs/v0.2-beta`) are excluded from the workflow. **`.github/workflows/docs-deploy.yml`** — deploys `docs-deployment` to the IC: @@ -72,7 +79,7 @@ Located at [docs-site/versions.json](../docs-site/versions.json). Update when re } ``` -The workflow copies this to the `docs-deployment` root and generates `index.html` redirecting to the version marked `latest: true`. +The workflow copies this to the `docs-deployment` root, generates `index.html` redirecting to the version marked `latest: true`, and copies that version's `llms.txt` to the root for agent discovery. ## Common Tasks @@ -124,17 +131,21 @@ git push origin docs/v0.1 Or push a patch tag (`v0.1.1`) — it deploys to the same `/0.1/` directory. -### Beta Versions +## Agent-Friendly Docs -Create a docs branch with the full version: +The site implements the [Agent-Friendly Documentation spec](https://agentdocsspec.com) so AI agents can discover and consume docs programmatically. -```bash -git checkout -b docs/v0.2.0-beta.5 -git push origin docs/v0.2.0-beta.5 -# → Deploys to /0.2.0-beta.5/ -``` +### Components + +- **`astro-agent-docs.mjs`** — Astro integration that generates clean `.md` endpoints (frontmatter stripped, title prepended) and `llms.txt` for each build +- **`rehype-agent-signaling.mjs`** — Injects a visually-hidden `
` on every HTML page pointing agents to `/llms.txt` +- **Root `llms.txt`** — Copied from the latest version's `llms.txt` by `publish-root-files`, with version navigation links prepended from `versions.json` + +### How it works + +Each versioned build produces its own `llms.txt` and `.md` files inside the version folder (e.g., `/0.2/llms.txt`, `/0.2/quickstart.md`). These are always in sync because they're generated from the same source in the same build. -Don't add beta versions to `versions.json` — they won't appear in the version switcher. +The root `/llms.txt` is assembled by `publish-root-files`: it fetches the latest version's `llms.txt` from the `docs-deployment` branch and prepends version navigation links. This ensures root `llms.txt` updates whenever `versions.json` changes. ## Local Testing