feat: clear meta in chunk

Author: darkskygit

packages/backend/server/src/models/common/copilot.ts

@@ -148,3 +148,36 @@ export type IgnoredDoc = {
   createdByAvatar: string | undefined;
   updatedBy: string | undefined;
 };
+
+export const EMBEDDING_DIMENSIONS = 1024;
+
+const FILTER_PREFIX = [
+  'Title: ',
+  'Created at: ',
+  'Updated at: ',
+  'Created by: ',
+  'Updated by: ',
+];
+
+export function clearEmbeddingContent(content: string): string {
+  const lines = content.split('\n');
+  let maxLines = 5;
+  while (maxLines > 0 && lines.length > 0) {
+    if (FILTER_PREFIX.some(prefix => lines[0].startsWith(prefix))) {
+      lines.shift();
+      maxLines--;
+    } else {
+      // only process consecutive metadata rows
+      break;
+    }
+  }
+  return lines.join('\n');
+}
+
+export function clearEmbeddingChunk(chunk: ChunkSimilarity): ChunkSimilarity {
+  if (chunk.content) {
+    const content = clearEmbeddingContent(chunk.content);
+    return { ...chunk, content };
+  }
+  return chunk;
+}

packages/backend/server/src/models/copilot-context.ts

@@ -6,21 +6,21 @@ import { Prisma } from '@prisma/client';
 import { CopilotSessionNotFound } from '../base';
 import { BaseModel } from './base';
 import {
+  clearEmbeddingContent,
   ContextBlob,
   ContextConfigSchema,
   ContextDoc,
   ContextEmbedStatus,
   CopilotContext,
   DocChunkSimilarity,
   Embedding,
+  EMBEDDING_DIMENSIONS,
   FileChunkSimilarity,
   MinimalContextConfigSchema,
 } from './common/copilot';
 
 type UpdateCopilotContextInput = Pick<CopilotContext, 'config'>;
 
-export const EMBEDDING_DIMENSIONS = 1024;
-
 /**
  * Copilot Job Model
  */
@@ -215,7 +215,7 @@ export class CopilotContextModel extends BaseModel {
       select: { content: true },
       orderBy: { chunk: 'asc' },
     });
-    return file?.map(f => f.content).join('\n');
+    return file?.map(f => clearEmbeddingContent(f.content)).join('\n');
   }
 
   async insertFileEmbedding(
@@ -274,7 +274,7 @@ export class CopilotContextModel extends BaseModel {
       select: { content: true },
       orderBy: { chunk: 'asc' },
     });
-    return file?.map(f => f.content).join('\n');
+    return file?.map(f => clearEmbeddingContent(f.content)).join('\n');
   }
 
   async insertWorkspaceEmbedding(

packages/backend/server/src/plugins/copilot/mcp/provider.ts

@@ -6,9 +6,9 @@ import z from 'zod';
 
 import { DocReader } from '../../../core/doc';
 import { AccessController } from '../../../core/permission';
+import { clearEmbeddingChunk } from '../../../models';
 import { IndexerService } from '../../indexer';
 import { CopilotContextService } from '../context';
-import { clearEmbeddingChunk } from '../utils';
 
 @Injectable()
 export class WorkspaceMcpProvider {

packages/backend/server/src/plugins/copilot/tools/doc-semantic-search.ts

@@ -3,11 +3,14 @@ import { omit } from 'lodash-es';
 import { z } from 'zod';
 
 import type { AccessController } from '../../../core/permission';
-import type { ChunkSimilarity, Models } from '../../../models';
+import {
+  type ChunkSimilarity,
+  clearEmbeddingChunk,
+  type Models,
+} from '../../../models';
 import type { CopilotContextService } from '../context';
 import type { ContextSession } from '../context/session';
 import type { CopilotChatOptions } from '../providers';
-import { clearEmbeddingChunk } from '../utils';
 import { toolError } from './error';
 
 export const buildDocSearchGetter = (

packages/backend/server/src/plugins/copilot/utils.ts

@@ -3,7 +3,6 @@ import { Readable } from 'node:stream';
 import type { Request } from 'express';
 
 import { OneMB, readBufferWithLimit } from '../../base';
-import type { ChunkSimilarity } from '../../models';
 import type { PromptTools } from './providers';
 import type { ToolsConfig } from './types';
 
@@ -83,29 +82,3 @@ export function getTools(
   });
   return result;
 }
-
-const FILTER_PREFIX = [
-  'Title: ',
-  'Created at: ',
-  'Updated at: ',
-  'Created by: ',
-  'Updated by: ',
-];
-
-export function clearEmbeddingChunk(chunk: ChunkSimilarity): ChunkSimilarity {
-  if (chunk.content) {
-    const lines = chunk.content.split('\n');
-    let maxLines = 5;
-    while (maxLines > 0 && lines.length > 0) {
-      if (FILTER_PREFIX.some(prefix => lines[0].startsWith(prefix))) {
-        lines.shift();
-        maxLines--;
-      } else {
-        // only process consecutive metadata rows
-        break;
-      }
-    }
-    return { ...chunk, content: lines.join('\n') };
-  }
-  return chunk;
-}