Omnibus AI enhancements

.gitignore

@@ -14,6 +14,11 @@ public/sitemap.xml
 # Scraped documentation for LLM training
 scraped-docs/
 /public/_scraped-docs
+/public/llms.txt
+/public/llms-full.txt
+
+# Per-page .md files generated at build time (Solana-style LLM access)
+/public/**/*.md
 public/_seo-audit.json
 src/vendor/bytebellai/index.js
 src/vendor/bytebellai/style.css

app/api/llm-md/[...path]/route.js

@@ -0,0 +1,112 @@
+import { NextResponse } from 'next/server';
+import fs from 'fs/promises';
+import path from 'path';
+
+export async function GET(request, { params }) {
+	const { path: segments } = await params;
+	const docPath = segments.join('/');
+
+	const contentDir = path.join(process.cwd(), 'content');
+	const candidates = [
+		path.join(contentDir, `${docPath}.mdx`),
+		path.join(contentDir, `${docPath}.md`),
+		path.join(contentDir, docPath, 'index.mdx'),
+		path.join(contentDir, docPath, 'index.md')
+	];
+
+	let raw = null;
+	for (const filePath of candidates) {
+		try {
+			raw = await fs.readFile(filePath, 'utf8');
+			break;
+		} catch {
+			continue;
+		}
+	}
+
+	if (!raw) {
+		return NextResponse.json({ error: 'Page not found' }, { status: 404 });
+	}
+
+	const markdown = mdxToMarkdown(raw, `https://docs.sei.io/${docPath}`);
+
+	return new NextResponse(markdown, {
+		headers: {
+			'Content-Type': 'text/markdown; charset=utf-8',
+			'X-Robots-Tag': 'noindex'
+		}
+	});
+}
+
+function mdxToMarkdown(source, url) {
+	let result = source;
+
+	// Extract and preserve frontmatter
+	let frontmatter = '';
+	const fmMatch = result.match(/^---\n([\s\S]*?)\n---/);
+	if (fmMatch) {
+		frontmatter = fmMatch[1];
+		result = result.slice(fmMatch[0].length);
+	}
+
+	// Strip import statements
+	result = result.replace(/^import\s+.*$/gm, '');
+
+	// Strip JSX component wrappers like <Callout ...> and </Callout> but keep inner text
+	result = result.replace(/<(Callout|Steps|Cards|Card|Tabs|Tab|FileTree)(\s[^>]*)?\/?>/gi, '');
+	result = result.replace(/<\/(Callout|Steps|Cards|Card|Tabs|Tab|FileTree)>/gi, '');
+
+	// Convert HTML tables to markdown tables
+	result = convertHtmlTablesToMarkdown(result);
+
+	// Strip remaining HTML/JSX wrapper tags (div, span, section) but keep content
+	result = result.replace(/<\/?(div|span|section|article|thead|tbody|main)(\s[^>]*)?>/gi, '');
+
+	// Clean up excessive blank lines
+	result = result.replace(/\n{4,}/g, '\n\n\n');
+	result = result.trim();
+
+	// Rebuild with frontmatter including the URL
+	let md = '---\n';
+	if (frontmatter) md += frontmatter + '\n';
+	md += `url: ${url}\n`;
+	md += '---\n\n';
+	md += result;
+
+	return md;
+}
+
+function convertHtmlTablesToMarkdown(text) {
+	return text.replace(/<table[^>]*>([\s\S]*?)<\/table>/gi, (_, tableContent) => {
+		const rows = [];
+		const rowMatches = tableContent.matchAll(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
+
+		for (const rowMatch of rowMatches) {
+			const cells = [];
+			const cellMatches = rowMatch[1].matchAll(/<(th|td)[^>]*>([\s\S]*?)<\/\1>/gi);
+			for (const cellMatch of cellMatches) {
+				const cellText = cellMatch[2]
+					.replace(/<[^>]*>/g, '')
+					.replace(/\s+/g, ' ')
+					.trim();
+				cells.push(cellText);
+			}
+			if (cells.length > 0) rows.push(cells);
+		}
+
+		if (rows.length === 0) return '';
+
+		let md = '';
+		const colCount = Math.max(...rows.map((r) => r.length));
+
+		rows.forEach((row, i) => {
+			const padded = Array.from({ length: colCount }, (_, j) => row[j] || '');
+			md += '| ' + padded.join(' | ') + ' |\n';
+			if (i === 0) {
+				md += '| ' + padded.map(() => '---').join(' | ') + ' |\n';
+			}
+		});
+
+		return '\n' + md;
+	});
+}

next.config.mjs

@@ -78,6 +78,15 @@ export default withNextra({
 					{ key: 'Vercel-CDN-Cache-Control', value: 'public, max-age=3600, immutable' }
 				]
 			},
+			{
+				source: '/(.*)\\.(md|txt)',
+				headers: [
+					{ key: 'Content-Type', value: 'text/markdown; charset=utf-8' },
+					{ key: 'Cache-Control', value: 'public, max-age=3600, s-maxage=86400, stale-while-revalidate=604800' },
+					{ key: 'Vercel-CDN-Cache-Control', value: 'public, max-age=3600, s-maxage=86400, stale-while-revalidate=604800' },
+					{ key: 'X-Robots-Tag', value: 'noindex' }
+				]
+			},
 			{
 				source: '/(.*)',
 				headers: [

package.json

@@ -43,8 +43,6 @@
 	"devDependencies": {
 		"@playwright/test": "^1.56.1",
 		"@react-native-async-storage/async-storage": "^2.1.2",
-		"@sparticuz/chromium": "^141.0.0",
-		"@sparticuz/chromium-min": "^141.0.0",
 		"@types/node": "18.11.10",
 		"@types/react": "^19.2.7",
 		"@types/react-dom": "^19.2.3",
@@ -66,8 +64,6 @@
 		"postcss": "^8.5.6",
 		"postcss-simple-vars": "^7.0.1",
 		"prettier": "^3.5.3",
-		"puppeteer": "^24.31.0",
-		"puppeteer-core": "^24.31.0",
 		"tailwindcss": "^4.1.18",
 		"terser": "^5.31.0",
 		"tsx": "^4.19.3",

proxy.ts

@@ -0,0 +1,19 @@
+import { NextResponse } from 'next/server';
+import type { NextRequest } from 'next/server';
+
+export function proxy(request: NextRequest) {
+	const { pathname } = request.nextUrl;
+
+	if (pathname.endsWith('.md')) {
+		const cleanPath = pathname.slice(0, -3);
+		const url = request.nextUrl.clone();
+		url.pathname = `/api/llm-md${cleanPath}`;
+		return NextResponse.rewrite(url);
+	}
+
+	return NextResponse.next();
+}
+
+export const config = {
+	matcher: ['/((?!api|_next|_pagefind|_scraped-docs|assets|favicon).+\\.md)']
+};