Add LLM-friendly documentation (llms.txt) (#824)

Author: goruha

.gitignore

@@ -107,3 +107,6 @@ changelog/*
 .atmos
 .python-version
 yarn.lock
+
+**/llms.txt
+**/llms-full.txt

docusaurus.config.js

@@ -100,6 +100,30 @@ async function createConfig() {
           redirects: getStaticRedirects(),
         },
       ],
+      [
+        'docusaurus-plugin-llms',
+        {
+          generateLLMsTxt: true,
+          generateLLMsFullTxt: true,
+          docsDir: 'docs',
+          includeBlog: true,
+          includeOrder: [
+            'into/*',
+            'quickstart/*',
+            'jumpstart/*',
+            'learn/*',
+            'layers/*',
+            'best-practices/*',
+            'reference/*',
+            'components/*',
+            'modules/*',
+            'github-actions/*',
+            'resources/*',
+            'community/*',
+            'support/*',
+          ],
+        },
+      ],
       redirectsPlugin,
       async function AddTailwindCSS(context, options) {
         return {
@@ -130,7 +154,7 @@ async function createConfig() {
             showLastUpdateAuthor: true,
             // Ignore tags missing from tags.yml
             // There are many from components and GHA, it's too much noise without being helpful
-            onInlineTags: 'ignore', 
+            onInlineTags: 'ignore',
             tags: 'tags.yml',
             include: ['**/*.md', '**/*.mdx']
           },

package-lock.json

@@ -6,6 +6,7 @@
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
     "build": "docusaurus build",
+    "postbuild": "cp build/llms.txt static/llms.txt && cp build/llms-full.txt static/llms-full.txt && ./scripts/clean-llms-imports.sh",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
@@ -71,7 +72,8 @@
     "sass": "^1.77.8",
     "styled-jsx": "^5.1.6",
     "unified": "^11.0.5",
-    "unist-util-visit": "^5.0.0"
+    "unist-util-visit": "^5.0.0",
+    "docusaurus-plugin-llms": "^0.2.2"
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.4.0",

package.json

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Script to clean React/MDX import statements from llms.txt files
+# These import statements are extracted by docusaurus-plugin-llms from MDX files
+# and need to be removed to keep the files as plain text for LLMs
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STATIC_DIR="$SCRIPT_DIR/../static"
+
+echo "Cleaning import statements from llms.txt files..."
+
+# Clean llms.txt - replace import statements with [Description not available]
+if [ -f "$STATIC_DIR/llms.txt" ]; then
+    sed -i.bak 's/: import [A-Za-z{}, ]* from ['\''"].*['\''"];*$/: [Description not available]/' "$STATIC_DIR/llms.txt"
+    rm "$STATIC_DIR/llms.txt.bak"
+    echo "✓ Cleaned $STATIC_DIR/llms.txt"
+fi
+
+# Clean llms-full.txt - remove import statement lines entirely
+if [ -f "$STATIC_DIR/llms-full.txt" ]; then
+    sed -i.bak '/^import [A-Za-z{}, ]* from ['\''"].*['\''"];*$/d' "$STATIC_DIR/llms-full.txt"
+    rm "$STATIC_DIR/llms-full.txt.bak"
+    echo "✓ Cleaned $STATIC_DIR/llms-full.txt"
+fi
+
+echo "✓ Import cleanup complete"