Skip to content

Commit db39605

Browse files
thedaviddiasthedaviddias
committed
feat: add llms.txt and llmx-full.txt
1 parent 4ae9189 commit db39605

File tree

5 files changed

+268
-99
lines changed

5 files changed

+268
-99
lines changed

‎app/[lang]/llms-full.txt/route.ts‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import fg from 'fast-glob';
2+
import matter from 'gray-matter';
3+
import { NextResponse } from 'next/server';
4+
import * as fs from 'node:fs/promises';
5+
import path from 'node:path';
6+
import { remark } from 'remark';
7+
import remarkGfm from 'remark-gfm';
8+
import remarkStringify from 'remark-stringify';
9+
10+
export const dynamic = 'force-dynamic';
11+
12+
// Regular expressions for cleaning up the content
13+
const IMPORT_REGEX = /import\s+?(?:(?:{[^}]*}|\*|\w+)\s+from\s+)?['"](.*?)['"];?\n?/g;
14+
const COMPONENT_USAGE_REGEX = /<[A-Z][a-zA-Z]*(?:\s+[^>]*)?(?:\/?>|>[^<]*<\/[A-Z][a-zA-Z]*>)/g;
15+
const NEXTRA_COMPONENT_REGEX = /<(?:Callout|Steps|Tabs|Tab|FileTree)[^>]*>[^<]*<\/(?:Callout|Steps|Tabs|Tab|FileTree)>/g;
16+
const MDX_EXPRESSION_REGEX = /{(?:[^{}]|{[^{}]*})*}/g;
17+
const EXPORT_REGEX = /export\s+(?:default\s+)?(?:const|let|var|function|class|interface|type)?\s+[a-zA-Z_$][0-9a-zA-Z_$]*[\s\S]*?(?:;|\n|$)/g;
18+
19+
async function processContent(content: string): Promise<string> {
20+
try {
21+
// Multi-step cleanup to handle different MDX constructs
22+
const cleanContent = content
23+
// Remove imports first
24+
.replace(IMPORT_REGEX, '')
25+
// Remove exports
26+
.replace(EXPORT_REGEX, '')
27+
// Remove Nextra components with their content
28+
.replace(NEXTRA_COMPONENT_REGEX, '')
29+
// Remove other React components
30+
.replace(COMPONENT_USAGE_REGEX, '')
31+
// Remove MDX expressions
32+
.replace(MDX_EXPRESSION_REGEX, '')
33+
// Clean up multiple newlines
34+
.replace(/\n{3,}/g, '\n\n')
35+
// Remove empty JSX expressions
36+
.replace(/{[\s]*}/g, '')
37+
// Clean up any remaining JSX-like syntax
38+
.replace(/<>[\s\S]*?<\/>/g, '')
39+
.replace(/{\s*\/\*[\s\S]*?\*\/\s*}/g, '')
40+
.trim();
41+
42+
// Simple markdown processing without MDX
43+
const file = await remark()
44+
.use(remarkGfm)
45+
.use(remarkStringify)
46+
.process(cleanContent);
47+
48+
return String(file);
49+
} catch (error) {
50+
console.error('Error processing content:', error);
51+
// If processing fails, return a basic cleaned version
52+
return content
53+
.replace(IMPORT_REGEX, '')
54+
.replace(COMPONENT_USAGE_REGEX, '')
55+
.replace(MDX_EXPRESSION_REGEX, '')
56+
.trim();
57+
}
58+
}
59+
60+
interface ProcessFileOptions {
61+
baseUrl: string;
62+
lang: string;
63+
}
64+
65+
async function processFile(file: string, options: ProcessFileOptions) {
66+
try {
67+
const fileContent = await fs.readFile(file);
68+
const { content, data } = matter(fileContent.toString());
69+
70+
// Get the filename without extension to use as fallback title
71+
const basename = path.basename(file, '.mdx');
72+
73+
// Extract category from file path
74+
const pathParts = path.dirname(file).split(path.sep);
75+
const category = pathParts.length > 3 && pathParts[3] ? pathParts[3] : 'general';
76+
77+
// Skip if the file is marked as hidden or draft
78+
if (data.draft || data.hidden) {
79+
return null;
80+
}
81+
82+
// Use filename as title if no title in frontmatter, and convert to Title Case
83+
const title = data.title || basename.split('-')
84+
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
85+
.join(' ');
86+
87+
const processed = await processContent(content);
88+
const patternUrl = new URL(
89+
`/${options.lang}/patterns/${category}/${basename}`,
90+
options.baseUrl
91+
).toString();
92+
93+
return `# ${category.toUpperCase()}: [${title}](${patternUrl})
94+
95+
${data.description || ''}
96+
97+
${processed}`;
98+
} catch (error) {
99+
console.error(`Error processing file ${file}:`, error);
100+
return null;
101+
}
102+
}
103+
104+
export async function GET(
105+
_request: Request,
106+
{ params }: { params: { lang: string } }
107+
) {
108+
try {
109+
// Get base URL and await params
110+
const [baseUrl, { lang }] = await Promise.all([
111+
Promise.resolve(
112+
process.env.NEXT_PUBLIC_VERCEL_URL
113+
? `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`
114+
: process.env.NODE_ENV === 'development'
115+
? 'http://localhost:3000'
116+
: ''
117+
),
118+
params
119+
]);
120+
121+
if (!baseUrl) {
122+
return NextResponse.json({ error: 'Base URL not configured' }, { status: 500 });
123+
}
124+
125+
// Get files and process them
126+
const files = await fg(['content/en/patterns/**/*.mdx']);
127+
const options: ProcessFileOptions = { baseUrl, lang };
128+
129+
const scanned = (await Promise.all(
130+
files.map(file => processFile(file, options))
131+
)).filter(Boolean);
132+
133+
if (!scanned.length) {
134+
return NextResponse.json({ error: 'No content found' }, { status: 404 });
135+
}
136+
137+
return new Response(scanned.join('\n\n'), {
138+
headers: {
139+
'Content-Type': 'text/plain',
140+
},
141+
});
142+
} catch (error) {
143+
console.error('Error generating LLM content:', error);
144+
return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
145+
}
146+
}

‎app/[lang]/llms.txt/route.ts‎

Lines changed: 102 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,117 @@
1-
import fg from 'fast-glob';
21
import matter from 'gray-matter';
3-
import * as fs from 'node:fs/promises';
2+
import { NextResponse } from 'next/server';
3+
import fs from 'node:fs';
44
import path from 'node:path';
5-
import { remark } from 'remark';
6-
import remarkGfm from 'remark-gfm';
7-
import remarkStringify from 'remark-stringify';
85

9-
export const revalidate = false;
6+
const contentDirectory = path.join(process.cwd(), 'content/en/patterns');
107

11-
// Regular expressions for cleaning up the content
12-
const IMPORT_REGEX = /import\s+?(?:(?:{[^}]*}|\*|\w+)\s+from\s+)?['"](.*?)['"];?\n?/g;
13-
const COMPONENT_USAGE_REGEX = /<[A-Z][a-zA-Z]*(?:\s+[^>]*)?(?:\/?>|>[^<]*<\/[A-Z][a-zA-Z]*>)/g;
14-
const NEXTRA_COMPONENT_REGEX = /<(?:Callout|Steps|Tabs|Tab|FileTree)[^>]*>[^<]*<\/(?:Callout|Steps|Tabs|Tab|FileTree)>/g;
15-
const MDX_EXPRESSION_REGEX = /{(?:[^{}]|{[^{}]*})*}/g;
16-
const EXPORT_REGEX = /export\s+(?:default\s+)?(?:const|let|var|function|class|interface|type)?\s+[a-zA-Z_$][0-9a-zA-Z_$]*[\s\S]*?(?:;|\n|$)/g;
8+
interface Pattern {
9+
category: string;
10+
title: string;
11+
summary: string;
12+
status: string;
13+
slug: string;
14+
}
1715

18-
export async function GET() {
19-
try {
20-
const files = await fg(['content/en/patterns/**/*.mdx']);
21-
22-
const scan = files.map(async (file) => {
23-
try {
24-
const fileContent = await fs.readFile(file);
25-
const { content, data } = matter(fileContent.toString());
26-
27-
// Get the filename without extension to use as fallback title
28-
const basename = path.basename(file, '.mdx');
29-
30-
// Extract category from file path
31-
const pathParts = path.dirname(file).split(path.sep);
32-
let category = 'general';
33-
if (pathParts.length > 3 && pathParts[3]) {
34-
category = pathParts[3];
35-
}
36-
37-
// Skip if the file is marked as hidden or draft
38-
if (data.draft || data.hidden) {
39-
return null;
40-
}
41-
42-
// Use filename as title if no title in frontmatter, and convert to Title Case
43-
const title = data.title || basename.split('-')
44-
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
45-
.join(' ');
46-
47-
const processed = await processContent(content);
48-
return `File: ${file}
49-
# ${category.toUpperCase()}: ${title}
50-
51-
${data.description || ''}
52-
53-
${processed}`;
54-
} catch (error) {
55-
console.error(`Error processing file ${file}:`, error);
56-
return null;
57-
}
58-
});
16+
interface PatternsByCategory {
17+
[category: string]: Pattern[];
18+
}
5919

60-
const scanned = (await Promise.all(scan)).filter(Boolean);
20+
function slugify(text: string): string {
21+
return text
22+
.toLowerCase()
23+
.replace(/[^a-z0-9]+/g, '-')
24+
.replace(/(^-|-$)/g, '');
25+
}
6126

62-
if (!scanned.length) {
63-
return new Response('No content found', { status: 404 });
64-
}
27+
function getAllPatterns(): PatternsByCategory {
28+
// Get all directories under patterns
29+
const categories = fs.readdirSync(contentDirectory);
6530

66-
return new Response(scanned.join('\n\n'));
67-
} catch (error) {
68-
console.error('Error generating LLM content:', error);
69-
return new Response('Internal Server Error', { status: 500 });
31+
const allPatterns: PatternsByCategory = {};
32+
33+
for (const category of categories) {
34+
const categoryPath = path.join(contentDirectory, category);
35+
36+
// Skip if not a directory
37+
if (!fs.statSync(categoryPath).isDirectory()) continue;
38+
39+
// Read all MDX files in the category
40+
const files = fs.readdirSync(categoryPath)
41+
.filter(file => file.endsWith('.mdx'));
42+
43+
const categoryPatterns = files.map(file => {
44+
const fullPath = path.join(categoryPath, file);
45+
const fileContents = fs.readFileSync(fullPath, 'utf8');
46+
const { data } = matter(fileContents);
47+
const slug = file.replace('.mdx', '');
48+
49+
return {
50+
category,
51+
title: data.title || slug,
52+
summary: data.summary || '',
53+
status: data.status || 'coming soon',
54+
slug
55+
};
56+
});
57+
58+
allPatterns[category] = categoryPatterns;
7059
}
60+
61+
return allPatterns;
7162
}
7263

73-
async function processContent(content: string): Promise<string> {
64+
export async function GET(
65+
request: Request,
66+
{ params }: { params: { lang: string } }
67+
) {
7468
try {
75-
// Multi-step cleanup to handle different MDX constructs
76-
let cleanContent = content
77-
// Remove imports first
78-
.replace(IMPORT_REGEX, '')
79-
// Remove exports
80-
.replace(EXPORT_REGEX, '')
81-
// Remove Nextra components with their content
82-
.replace(NEXTRA_COMPONENT_REGEX, '')
83-
// Remove other React components
84-
.replace(COMPONENT_USAGE_REGEX, '')
85-
// Remove MDX expressions
86-
.replace(MDX_EXPRESSION_REGEX, '')
87-
// Clean up multiple newlines
88-
.replace(/\n{3,}/g, '\n\n')
89-
// Remove empty JSX expressions
90-
.replace(/{[\s]*}/g, '')
91-
// Clean up any remaining JSX-like syntax
92-
.replace(/<>[\s\S]*?<\/>/g, '')
93-
.replace(/{\s*\/\*[\s\S]*?\*\/\s*}/g, '')
94-
.trim();
95-
96-
// Simple markdown processing without MDX
97-
const file = await remark()
98-
.use(remarkGfm)
99-
.use(remarkStringify)
100-
.process(cleanContent);
101-
102-
return String(file);
69+
const patterns = getAllPatterns();
70+
71+
// Get base URL and await params
72+
const [baseUrl, { lang }] = await Promise.all([
73+
Promise.resolve(
74+
process.env.NODE_ENV === 'development'
75+
? 'http://localhost:3000'
76+
: `https://${process.env.NEXT_PUBLIC_VERCEL_URL || 'localhost:3000'}`
77+
),
78+
params
79+
]);
80+
81+
// Generate the text content
82+
let content = `# UX Patterns for Developers
83+
84+
## Overview
85+
This is an automatically generated overview of all UX patterns documented in this project.
86+
87+
## Pattern Categories\n`;
88+
89+
// Add patterns by category
90+
for (const [category, categoryPatterns] of Object.entries(patterns)) {
91+
content += `\n### ${category.charAt(0).toUpperCase() + category.slice(1)}\n`;
92+
for (const pattern of categoryPatterns) {
93+
const patternUrl = `${baseUrl}/${lang}/patterns/${category}/${pattern.slug}`;
94+
content += `- [${pattern.title}](${patternUrl})${pattern.summary ? `: ${pattern.summary}` : ''} [${pattern.status}]\n`;
95+
}
96+
}
97+
98+
content += `\n## Additional Resources
99+
- [Blog posts and articles about UX patterns](${baseUrl}/${lang}/blog)
100+
- [Comprehensive glossary of UX terms](${baseUrl}/${lang}/glossary)
101+
102+
## Technical Implementation
103+
- Built with Next.js and TypeScript
104+
- MDX-based pattern documentation
105+
- Accessibility-first approach
106+
- Comprehensive testing guidelines`;
107+
108+
return new NextResponse(content, {
109+
headers: {
110+
'Content-Type': 'text/plain',
111+
},
112+
});
103113
} catch (error) {
104-
console.error('Error processing content:', error);
105-
// If processing fails, return a basic cleaned version
106-
return content
107-
.replace(IMPORT_REGEX, '')
108-
.replace(COMPONENT_USAGE_REGEX, '')
109-
.replace(MDX_EXPRESSION_REGEX, '')
110-
.trim();
114+
console.error('Error generating content:', error);
115+
return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
111116
}
112117
}

‎app/_constants/footer.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,20 @@ export const FOOTER_MENU_LINKS = (lang: string) => [
1111
path: `/${lang}/blog`,
1212
label: 'Blog'
1313
},
14+
{
15+
path: `/${lang}/llms.txt`,
16+
label: 'LLMs'
17+
},
18+
{
19+
path: `/${lang}/llms-full.txt`,
20+
label: 'LLMs Full'
21+
},
1422
{
1523
path: `/${lang}/about`,
1624
label: 'About'
1725
},
1826
{
1927
path: `/${lang}/privacy-policy`,
2028
label: 'Privacy Policy'
21-
},
29+
}
2230
]

‎middleware.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ export { middleware } from 'nextra/locales'
33
export const config = {
44
// Matcher ignoring `/_next/` and `/api/`
55
matcher: [
6-
'/((?!api/mdx|api/email|api/patterns/random|api/og|_next/static|_next/image|llms.txt|favicon.ico|robots.txt|og/opengraph-image.png|covers|twitter-image|sitemap.xml|6ba7b811-9dad-11d1-80b4.txt|43mg4ybv6sxxanu24g7dngawd9up5w93.txt|apple-icon.png|manifest|_pagefind|examples).*)'
6+
'/((?!api/mdx|api/email|api/patterns/random|api/og|_next/static|_next/image|llms-full.txt|llms.txt|favicon.ico|robots.txt|og/opengraph-image.png|covers|twitter-image|sitemap.xml|6ba7b811-9dad-11d1-80b4.txt|43mg4ybv6sxxanu24g7dngawd9up5w93.txt|apple-icon.png|manifest|_pagefind|examples).*)'
77
]
88
}

0 commit comments

Comments
 (0)