improve caching

Author: arvinxx

packages/file-loaders/src/loaders/excel/__snapshots__/index.test.ts.snap

@@ -1,27 +1,25 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`ExcelLoader > should aggregate content correctly (joining sheets) > aggregated_content 1`] = `
-"## Sheet: 表1
-
+"<sheet name="表1" index="0">
 | __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
 | --- | --- | --- | --- | --- |
 | 项目 1 | 5 | 7 |  |  |
 | 项目 2 | 10 | 8 |  |  |
 | 项目 3 | 9 | 15 |  |  |
 | 项目 4 | 7 | 12 |  |  |
 | 项目 5 | 16 | 21 |  |  |
+</sheet>
 
----
-
-## Sheet: 表2 - 表格 2
-
+<sheet name="表2 - 表格 2" index="1">
 | __EMPTY | 类别 A | 类别 B | __EMPTY_1 | __EMPTY_2 |
 | --- | --- | --- | --- | --- |
 | 项目 1 | 5 | 7 |  |  |
 | 项目 2 | 10 | 8 |  |  |
 | 项目 3 | 9 | 15 |  |  |
 | 项目 4 | 7 | 12 |  |  |
-| 项目 5 | 16 | 21 |  |  |"
+| 项目 5 | 16 | 21 |  |  |
+</sheet>"
 `;
 
 exports[`ExcelLoader > should load pages correctly from an Excel file (one page per sheet) 1`] = `

packages/file-loaders/src/loaders/excel/index.ts

@@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises';
 import * as xlsx from 'xlsx';
 
 import type { DocumentPage, FileLoaderInterface } from '../../types';
+import { promptTemplate } from './prompt';
 
 const log = debug('file-loaders:excel');
 
@@ -135,13 +136,7 @@ export class ExcelLoader implements FileLoaderInterface {
    */
   async aggregateContent(pages: DocumentPage[]): Promise<string> {
     log('Aggregating content from', pages.length, 'Excel pages');
-    const result = pages
-      .map((page) => {
-        const sheetName = page.metadata.sheetName;
-        const header = sheetName ? `## Sheet: ${sheetName}\n\n` : '';
-        return header + page.pageContent;
-      })
-      .join('\n\n---\n\n'); // Separator between sheets
+    const result = promptTemplate(pages);
 
     log('Excel content aggregated successfully, length:', result.length);
     return result;

packages/file-loaders/src/loaders/excel/prompt.ts

@@ -0,0 +1,18 @@
+import type { DocumentPage } from '../../types';
+
+export const promptTemplate = (pages: DocumentPage[]) => {
+  return (
+    pages
+      .map((page, index) => {
+        const sheetName = page.metadata.sheetName;
+
+        const sheetIndex = page.metadata?.pageNumber || index;
+
+        return `<sheet name="${sheetName}" index="${sheetIndex}">
+${page.pageContent}
+</sheet>`;
+      })
+      // Separator between sheets
+      .join('\n\n')
+  );
+};

packages/file-loaders/src/loaders/pdf/__snapshots__/index.test.ts.snap

@@ -1,7 +1,8 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`PdfLoader > should aggregate content correctly 1`] = `
-"简单报告
+"<page pageNumber="1">
+简单报告
 副标题
 轻点或点按此占位符⽂本并开始键⼊即可开始。你可以在 Mac、iPad、iPhone 或
 iCloud.com 上查看和编辑此⽂稿。
@@ -24,10 +25,13 @@ Pages ⽂稿可⽤于⽂字处理和⻚⾯布局。此“简单报告”模板
 ⽂本添加你⾃⼰的内容。”
 ⻚脚 
 1
+</page>
 
+<page pageNumber="2">
 这是第⼆⻚的内容
 ⻚脚 
-2"
+2
+</page>"
 `;
 
 exports[`PdfLoader > should attach document metadata correctly 1`] = `

packages/file-loaders/src/loaders/pdf/index.ts

@@ -7,6 +7,7 @@ import * as _pdfjsWorker from 'pdfjs-dist/legacy/build/pdf.worker.mjs';
 import type { TextContent } from 'pdfjs-dist/types/src/display/api';
 
 import type { DocumentPage, FileLoaderInterface } from '../../types';
+import { promptTemplate } from './prompt';
 
 const log = debug('file-loaders:pdf');
 
@@ -132,7 +133,7 @@ export class PdfLoader implements FileLoaderInterface {
       `Found ${validPages.length} valid pages for aggregation (${pages.length - validPages.length} pages with errors filtered out)`,
     );
 
-    const result = validPages.map((page) => page.pageContent).join('\n\n');
+    const result = promptTemplate(validPages);
     log('PDF content aggregated successfully, length:', result.length);
     return result;
   }

packages/file-loaders/src/loaders/pdf/prompt.ts

@@ -0,0 +1,13 @@
+import type { DocumentPage } from '../../types';
+
+export const promptTemplate = (pages: DocumentPage[]) => {
+  return pages
+    .map((page, index) => {
+      const pageNumber = page.metadata?.pageNumber || index;
+
+      return `<page pageNumber="${pageNumber}">
+${page.pageContent}
+</page>`;
+    })
+    .join('\n\n');
+};

packages/file-loaders/test/__snapshots__/loaders.test.ts.snap

@@ -2,7 +2,9 @@
 
 exports[`loadFile Integration Tests > PDF Handling > should load content from a pdf file using filePath 1`] = `
 {
-  "content": "123",
+  "content": "<page pageNumber="1">
+123
+</page>",
   "fileType": "pdf",
   "filename": "test.pdf",
   "metadata": {

packages/file-loaders/test/loaders.test.ts

@@ -44,7 +44,7 @@ describe('loadFile Integration Tests', () => {
       // Pass filePath directly to loadFile
       const docs = await loadFile(filePath);
 
-      expect(docs.content).toEqual('123');
+      expect(docs.content).toContain('123');
       expect(docs.source).toEqual(filePath);
 
       // @ts-expect-error