Skip to content

Commit b0bf5dc

Browse files
dreamer6680dreamer6680
committed
fix: resole crawl cannot get docs
1 parent 061547a commit b0bf5dc

File tree

29 files changed

+358
-421
lines changed

29 files changed

+358
-421
lines changed

‎document/app/[lang]/robots.txt/route.ts

Lines changed: 32 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,71 +1,64 @@
11
import * as fs from 'node:fs/promises';
2+
import * as path from 'node:path';
23
import fg from 'fast-glob';
34
import matter from 'gray-matter';
45
import { i18n } from '@/lib/i18n';
56

67
export const revalidate = false;
78

8-
// 将文件路径转换为URL路径
9+
// 黑名单路径（不带语言前缀）
10+
const blacklist = ['use-cases/index', 'protocol/index', 'api/index'];
11+
12+
// 将文件路径转换为 URL 路径（包括文件名）
913
function filePathToUrl(filePath: string, defaultLanguage: string): string {
10-
// 移除 ./content/docs/ 前缀
11-
let urlPath = filePath.replace('./content/docs/', '');
12-
13-
// 确定基础路径
14+
let relativePath = filePath.replace('./content/docs/', '');
15+
1416
const basePath = defaultLanguage === 'zh-CN' ? '/docs' : '/en/docs';
15-
16-
// 如果是英文文件，移除 .en 后缀
17-
if (defaultLanguage !== 'zh-CN' && urlPath.endsWith('.en.mdx')) {
18-
urlPath = urlPath.replace('.en.mdx', '');
19-
} else if (urlPath.endsWith('.mdx')) {
20-
urlPath = urlPath.replace('.mdx', '');
21-
}
22-
23-
// 处理 index 文件
24-
if (urlPath.endsWith('/index')) {
25-
urlPath = urlPath.replace('/index', '');
17+
18+
if (defaultLanguage !== 'zh-CN' && relativePath.endsWith('.en.mdx')) {
19+
relativePath = relativePath.replace(/\.en\.mdx$/, '');
20+
} else if (relativePath.endsWith('.mdx')) {
21+
relativePath = relativePath.replace(/\.mdx$/, '');
2622
}
27-
28-
// 拼接完整路径
29-
return `${basePath}/${urlPath}`.replace(/\/\/+/g, '/');
23+
24+
return `${basePath}/${relativePath}`.replace(/\/\/+/g, '/');
25+
}
26+
27+
// 判断是否为黑名单路径
28+
function isBlacklisted(url: string): boolean {
29+
return blacklist.some(
30+
(item) => url.endsWith(`/docs/${item}`) || url.endsWith(`/en/docs/${item}`)
31+
);
3032
}
3133

3234
export async function GET(request: Request) {
3335
const defaultLanguage = i18n.defaultLanguage;
34-
35-
// 检查请求路径是否为 /en/robots
36+
3637
const requestUrl = new URL(request.url);
3738
const isEnRobotsRoute = requestUrl.pathname === '/en/robots';
3839

3940
let globPattern;
40-
4141
if (isEnRobotsRoute) {
42-
// 如果是 /en/robots 路由，只选择 .en.mdx 文件
4342
globPattern = ['./content/docs/**/*.en.mdx'];
4443
} else if (defaultLanguage === 'zh-CN') {
45-
// 中文环境下的普通路由
4644
globPattern = ['./content/docs/**/*.mdx'];
4745
} else {
48-
// 英文环境下的普通路由
4946
globPattern = ['./content/docs/**/*.en.mdx'];
5047
}
5148

52-
const files = await fg(globPattern);
49+
const files = await fg(globPattern, { caseSensitiveMatch: true });
5350

54-
const urls = await Promise.all(
55-
files.map(async (file: string) => {
56-
const urlPath = filePathToUrl(file, defaultLanguage);
57-
return `${urlPath}`;
58-
})
59-
);
51+
// 转换文件路径为 URL，并过滤黑名单
52+
const urls = files
53+
.map((file) => filePathToUrl(file, defaultLanguage))
54+
.filter((url) => !isBlacklisted(url));
6055

61-
// 按URL排序
6256
urls.sort((a, b) => a.localeCompare(b));
6357

64-
// 生成HTML链接列表
6558
const html = `
6659
<html>
6760
<head>
68-
<title>FastGPT Documentation Links</title>
61+
<title>FastGPT 文档目录</title>
6962
<style>
7063
body { font-family: Arial, sans-serif; margin: 20px; }
7164
h1 { color: #333; }
@@ -78,15 +71,15 @@ export async function GET(request: Request) {
7871
<body>
7972
<h1>Documentation Links</h1>
8073
<ul>
81-
${urls.map(url => `<li><a href="${url}">${url}</a></li>`).join('')}
74+
${urls.map((url) => `<li><a href="${url}">${url}</a></li>`).join('')}
8275
</ul>
8376
</body>
8477
</html>
8578
`;
8679

8780
return new Response(html, {
8881
headers: {
89-
'Content-Type': 'text/html',
90-
},
82+
'Content-Type': 'text/html'
83+
}
9184
});
92-
}
85+
}

‎document/app/api/meta/route.ts

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import type { NextRequest } from 'next/server';
2+
import { NextResponse } from 'next/server';
3+
import fs from 'fs/promises';
4+
import path from 'path';
5+
6+
const docsRoot = path.resolve(process.cwd(), 'content/docs');
7+
8+
function isInvalidPage(str: string): boolean {
9+
if (!str || typeof str !== 'string') return true;
10+
if (/\[.*?\]\(.*?\)/.test(str) || /^https?:\/\//.test(str) || /[()]/.test(str)) return true;
11+
if (/^\s*---[\s\S]*---\s*$/.test(str)) return true;
12+
return false;
13+
}
14+
15+
function getPageName(str: string): string {
16+
return str.startsWith('...') ? str.slice(3) : str;
17+
}
18+
19+
async function findFirstValidPage(dirRelPath: string): Promise<string | null> {
20+
const absDir = path.join(docsRoot, dirRelPath);
21+
const metaPath = path.join(absDir, 'meta.json');
22+
23+
try {
24+
const metaRaw = await fs.readFile(metaPath, 'utf-8');
25+
const meta = JSON.parse(metaRaw);
26+
if (!Array.isArray(meta.pages)) return null;
27+
28+
for (const page of meta.pages) {
29+
if (isInvalidPage(page)) continue;
30+
31+
const pageName = getPageName(page);
32+
const pagePath = path.join(dirRelPath, pageName);
33+
34+
const candidateDir = path.join(docsRoot, pagePath);
35+
const candidateFile = candidateDir + '.mdx';
36+
37+
try {
38+
await fs.access(candidateFile);
39+
return pagePath;
40+
} catch {
41+
try {
42+
const stat = await fs.stat(candidateDir);
43+
if (stat.isDirectory()) {
44+
const recursiveResult = await findFirstValidPage(pagePath);
45+
if (recursiveResult) return recursiveResult;
46+
}
47+
} catch {
48+
// ignore
49+
}
50+
}
51+
}
52+
} catch {
53+
// ignore
54+
}
55+
56+
return null;
57+
}
58+
59+
export async function GET(req: NextRequest) {
60+
const url = new URL(req.url);
61+
const rawPath = url.searchParams.get('path');
62+
63+
if (!rawPath || !rawPath.startsWith('/docs')) {
64+
return NextResponse.json({ error: 'Invalid path' }, { status: 400 });
65+
}
66+
67+
// 去除 /docs 前缀，且清理首尾斜杠
68+
const relPath = rawPath.replace(/^\/docs\/?/, '').replace(/^\/|\/$/g, '');
69+
70+
try {
71+
// 先检测是否有该 mdx 文件
72+
const maybeFile = path.join(docsRoot, relPath + '.mdx');
73+
await fs.access(maybeFile);
74+
// 如果存在，返回完整路径（带 /docs）
75+
return NextResponse.json('/docs/' + relPath);
76+
} catch {
77+
// 不存在，尝试递归寻找第一个有效页面
78+
const found = await findFirstValidPage(relPath);
79+
if (found) {
80+
// 返回带 /docs 前缀的完整路径
81+
return NextResponse.json('/docs/' + found.replace(/\\/g, '/'));
82+
} else {
83+
return NextResponse.json({ error: 'No valid mdx page found' }, { status: 404 });
84+
}
85+
}
86+
}

‎document/components/docs/not-found.tsx

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
'use client';
2-
import { redirect } from 'next/navigation';
3-
import { usePathname } from 'next/navigation';
42
import { useEffect } from 'react';
3+
import { usePathname, useRouter } from 'next/navigation';
54

65
const exactMap: Record<string, string> = {
76
'/docs/intro': '/docs/introduction',
@@ -21,25 +20,50 @@ const prefixMap: Record<string, string> = {
2120
'/docs/agreement': '/docs/protocol'
2221
};
2322

23+
const fallbackRedirect = '/docs/introduction';
24+
2425
export default function NotFound() {
2526
const pathname = usePathname();
27+
const router = useRouter();
2628

2729
useEffect(() => {
28-
if (exactMap[pathname]) {
29-
redirect(exactMap[pathname]);
30-
return;
31-
}
32-
33-
for (const [oldPrefix, newPrefix] of Object.entries(prefixMap)) {
34-
if (pathname.startsWith(oldPrefix)) {
35-
const rest = pathname.slice(oldPrefix.length);
36-
redirect(newPrefix + rest);
30+
const tryRedirect = async () => {
31+
if (exactMap[pathname]) {
32+
router.replace(exactMap[pathname]);
3733
return;
3834
}
39-
}
4035

41-
redirect('/docs/introduction');
42-
}, [pathname]);
36+
for (const [oldPrefix, newPrefix] of Object.entries(prefixMap)) {
37+
if (pathname.startsWith(oldPrefix)) {
38+
const rest = pathname.slice(oldPrefix.length);
39+
router.replace(newPrefix + rest);
40+
return;
41+
}
42+
}
43+
44+
try {
45+
const basePath = pathname.replace(/\/$/, '');
46+
const res = await fetch(`/api/meta?path=${basePath}`);
47+
console.log('res', res);
48+
49+
if (!res.ok) throw new Error('meta API not found');
50+
51+
const validPage = await res.json();
52+
53+
if (validPage) {
54+
console.log('validPage', validPage);
55+
router.replace(validPage);
56+
return;
57+
}
58+
} catch (e) {
59+
console.warn('meta.json fallback failed:', e);
60+
}
61+
62+
router.replace(fallbackRedirect);
63+
};
64+
65+
tryRedirect();
66+
}, [pathname, router]);
4367

44-
return <></>;
68+
return null;
4569
}

‎document/content/docs/api/index.mdx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
2-
title: API手册
3-
description: FastGPT API手册
2+
title: API 文档
3+
description: API 文档
44
---
55

66
import { Redirect } from '@/components/docs/Redirect';

‎document/content/docs/index.mdx

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎document/content/docs/introduction/FAQ/index.mdx

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎document/content/docs/introduction/development/custom-models/index.mdx

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎document/content/docs/introduction/development/design/index.mdx

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎document/content/docs/introduction/development/index.mdx

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎document/content/docs/introduction/development/migration/index.mdx

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)