docs: rework llms.txt and add .md responses

docs/site/app/lib/llms-utils.ts

@@ -0,0 +1,37 @@
+import * as fs from "node:fs/promises";
+import fg from "fast-glob";
+import matter from "gray-matter";
+import { remark } from "remark";
+import remarkStringify from "remark-stringify";
+import remarkMdx from "remark-mdx";
+
+export const DEFAULT_IGNORED_FILES = [
+  "!./content/docs/acknowledgments.mdx",
+  "!./content/docs/community.mdx",
+  "!./content/docs/telemetry.mdx",
+];
+
+export async function scanDocumentationFiles(
+  patterns: Array<string> = ["./content/docs/**/*.mdx"],
+  ignorePatterns: Array<string> = DEFAULT_IGNORED_FILES
+) {
+  return fg([...patterns, ...ignorePatterns]);
+}
+
+export async function parseFileContent(filePath: string) {
+  const fileContent = await fs.readFile(filePath);
+  return matter(fileContent.toString());
+}
+
+export async function processMarkdownContent(content: string): Promise<string> {
+  const file = await remark()
+    .use(remarkMdx)
+    .use(remarkStringify)
+    .process(content);
+
+  return String(file);
+}
+
+export function formatFilePath(filePath: string): string {
+  return filePath.replace("./content/docs", "").replace(/\.mdx$/, ".md");
+}

docs/site/app/llms-full.txt/route.ts

@@ -0,0 +1,28 @@
+// This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms.
+
+import {
+  scanDocumentationFiles,
+  parseFileContent,
+  processMarkdownContent,
+  formatFilePath,
+} from "../lib/llms-utils";
+
+export const revalidate = false;
+
+export async function GET(): Promise<Response> {
+  // all scanned content
+  const files = await scanDocumentationFiles();
+
+  const scan = files.map(async (file) => {
+    const { content, data } = await parseFileContent(file);
+
+    const processed = await processMarkdownContent(content);
+    return `- [${data.title}](${formatFilePath(file)}): ${data.description}
+
+   ${processed}`;
+  });
+
+  const scanned = await Promise.all(scan);
+
+  return new Response(scanned.join("\n\n"));
+}

docs/site/app/llms.md/[[...slug]]/route.ts

@@ -0,0 +1,33 @@
+// This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms.
+
+import { notFound } from "next/navigation";
+import { type NextRequest } from "next/server";
+import { repoDocsPages } from "../../source";
+import { parseFileContent, processMarkdownContent } from "../../lib/llms-utils";
+
+export const revalidate = false;
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: Promise<{ slug?: Array<string> }> }
+) {
+  const { slug } = await params;
+  const page = repoDocsPages.getPage(slug);
+  if (!page) notFound();
+
+  const { data, content } = await parseFileContent(
+    page.data._file.absolutePath
+  );
+  const txt = await processMarkdownContent(content);
+
+  const header = `# ${data.title}
+Description: ${data.description}
+
+`;
+
+  return new Response(header.concat(txt));
+}
+
+export function generateStaticParams() {
+  return repoDocsPages.generateParams();
+}

docs/site/app/llms.txt/route.ts

@@ -1,44 +1,38 @@
 // This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms.
 
-import * as fs from "node:fs/promises";
-import fg from "fast-glob";
-import matter from "gray-matter";
-import { remark } from "remark";
-import remarkStringify from "remark-stringify";
-import remarkMdx from "remark-mdx";
+import {
+  scanDocumentationFiles,
+  parseFileContent,
+  formatFilePath,
+} from "../lib/llms-utils";
+import { PRODUCT_SLOGANS } from "../../lib/constants";
 
 export const revalidate = false;
 
 export async function GET(): Promise<Response> {
   // all scanned content
-  const files = await fg([
-    "./content/docs/**/*.mdx",
-    "!./content/docs/acknowledgments.mdx",
-    "!./content/docs/community.mdx",
-    "!./content/docs/telemetry.mdx",
-  ]);
-
-  const scan = files.map(async (file) => {
-    const fileContent = await fs.readFile(file);
-    const { content, data } = matter(fileContent.toString());
-
-    const processed = await processContent(content);
-    return `file: ${file}
-   meta: ${JSON.stringify(data, null, 2)}
-
-   ${processed}`;
+  const files = await scanDocumentationFiles();
+
+  const scan = files.sort().map(async (file) => {
+    const { data } = await parseFileContent(file);
+
+    return `- [${data.title}](${formatFilePath(file)}): ${data.description}`;
   });
 
   const scanned = await Promise.all(scan);
 
-  return new Response(scanned.join("\n\n"));
-}
+  const header = `
+# Turborepo documentation
+
+Generated at: ${new Date().toUTCString()}
+
+## Turborepo
+
+> ${PRODUCT_SLOGANS.turbo}
+
+## Docs
 
-async function processContent(content: string): Promise<string> {
-  const file = await remark()
-    .use(remarkMdx)
-    .use(remarkStringify)
-    .process(content);
+`;
 
-  return String(file);
+  return new Response(header.concat(scanned.join("\n")));
 }

docs/site/next.config.ts

@@ -6,6 +6,11 @@ import { REDIRECTS_FOR_V2_DOCS } from "./lib/redirects/v2-docs.mjs";
 const withMDX = createMDX();
 const vercelToolbar = withVercelToolbar();
 
+const llmMarkdownRedirects = {
+  source: "/docs/:path*.md",
+  destination: "/llms.md/:path*",
+};
+
 const config: NextConfig = {
   reactStrictMode: true,
   images: {
@@ -40,8 +45,9 @@ const config: NextConfig = {
                 source: "/api/feedback",
                 destination: "https://vercel.com/api/feedback",
               },
+              llmMarkdownRedirects,
             ]
-          : undefined,
+          : [llmMarkdownRedirects],
     };
   },
   // Next.js still expects these to return Promises even without await