fix: Properly handle LLM Ref

This LLM sometimes adds `REF.../REF` tags that should be removed

Author: paultranvan

packages/cozy-search/src/components/Conversations/ChatItemLabel.jsx

@@ -2,25 +2,11 @@ import React from 'react'
 
 import Markdown from 'cozy-ui/transpiled/react/Markdown'
 
-/**
- * Sanitize chat content by removing special sources tags like
- * [REF]...[/REF] or [doc_X] that are not currently handled.
- *
- * @param {string} content - content to sanitize
- * @returns {string} sanitized content
- */
-const sanitizeContent = content => {
-  if (!content) {
-    return ''
-  }
-  return content
-    .replace(/\s?\[REF\][\s\S]*?\[\/REF\]/g, '')
-    .replace(/\s?\[doc_\d+\]/g, '')
-}
+import { sanitizeChatContent } from '../helpers'
 
 const ChatItemLabel = ({ label }) => {
   if (typeof label === 'string') {
-    const content = sanitizeContent(label)
+    const content = sanitizeChatContent(label)
     return <Markdown content={content} />
   }
 

packages/cozy-search/src/components/helpers.js

@@ -23,3 +23,25 @@ export const isMessageForThisConversation = (res, messagesId) =>
   messagesId.includes(res._id)
 
 export const isAssistantEnabled = () => flag('cozy.assistant.enabled')
+
+/**
+ * Sanitize chat content by removing special sources tags like
+ * [REF]...[/REF] or [doc_X] that are not currently handled.
+ *
+ * @param {string} content - content to sanitize
+ * @returns {string} sanitized content
+ */
+export const sanitizeChatContent = content => {
+  if (!content) {
+    return ''
+  }
+  return (
+    content
+      // Remove REFdoc_1/REF
+      .replace(/\s?\[REF\][\s\S]*?\[\/REF\]/g, '')
+      // Remove [REF]doc_1[/REF]
+      .replace(/\s?REF[\s\S]*?\/REF/g, '')
+      // remove « [doc_1] »
+      .replace(/\s?\[doc_\d+\]/g, '')
+  )
+}

packages/cozy-search/src/components/helpers.spec.js

@@ -0,0 +1,43 @@
+import { sanitizeChatContent } from './helpers'
+
+describe('sanitizeChatContent', () => {
+  it('should return empty string for empty content', () => {
+    expect(sanitizeChatContent('')).toBe('')
+    expect(sanitizeChatContent(null)).toBe('')
+    expect(sanitizeChatContent(undefined)).toBe('')
+  })
+
+  it('should return string if no special tags', () => {
+    const text = 'How you doing, here no ref.'
+    expect(sanitizeChatContent(text)).toBe(text)
+  })
+
+  it('should remove tags REF.../REF', () => {
+    const text = 'Before REFdoc_1/REF after'
+    expect(sanitizeChatContent(text)).toBe('Before after')
+  })
+
+  it('should remove tags [REF]...[/REF]', () => {
+    const text = 'Before [REF]doc_1[/REF] after'
+    expect(sanitizeChatContent(text)).toBe('Before after')
+  })
+
+  it('should remove mixed REF tags ', () => {
+    const text = 'A REF.../REF B [REF]...[/REF] C REF.../REF D'
+    expect(sanitizeChatContent(text)).toBe('A B C D')
+  })
+
+  it('should remove [tags]', () => {
+    const text = 'Here is a doc [doc_42] and some texte'
+    expect(sanitizeChatContent(text)).toBe('Here is a doc and some texte')
+  })
+
+  it('should not remove simple REF or [REF]', () => {
+    const text = 'REF not closed [REF]not closed either'
+    expect(sanitizeChatContent(text)).toBe(text)
+  })
+  it('should not remove lowercase ref', () => {
+    const text = 'before refdoc_1/ref after'
+    expect(sanitizeChatContent(text)).toBe(text)
+  })
+})