Add fuzzy filtering to the evaluation reports

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/App.tsx

@@ -3,7 +3,7 @@
 
 import { useState } from 'react';
 import { Settings28Regular, FilterDismissRegular, DismissRegular, ArrowDownloadRegular } from '@fluentui/react-icons';
-import { Button, Drawer, DrawerBody, DrawerHeader, DrawerHeaderTitle, Switch, Tooltip } from '@fluentui/react-components';
+import { Button, Drawer, DrawerBody, DrawerHeader, DrawerHeaderTitle, SearchBox, Switch, Tooltip } from '@fluentui/react-components';
 import { makeStyles } from '@fluentui/react-components';
 import './App.css';
 import { ScenarioGroup } from './ScenarioTree';
@@ -47,7 +47,7 @@ const useStyles = makeStyles({
 
 function App() {
   const classes = useStyles();
-  const { dataset, scoreSummary, selectedTags, clearFilters } = useReportContext();
+  const { dataset, scoreSummary, selectedTags, clearFilters, searchValue, setSearchValue } = useReportContext();
   const [isSettingsOpen, setIsSettingsOpen] = useState(false);
   const { renderMarkdown, setRenderMarkdown } = useReportContext();
   const { globalTags, filterableTags } = categorizeAndSortTags(dataset, scoreSummary.primaryResult.executionName);
@@ -77,11 +77,14 @@ function App() {
         <div className={classes.headerTop}>
           <h1>AI Evaluation Report</h1>
           <div className={classes.headerActions}>
-            {selectedTags.length > 0 && (
+            {(selectedTags.length > 0 || !!searchValue) && (
               <Tooltip content="Clear Filters" relationship="description">
                 <Button icon={<FilterDismissRegular />} appearance="subtle" onClick={clearFilters} />
               </Tooltip>
             )}
+            <SearchBox placeholder="Search / Filter " value={searchValue} type="text" 
+              style={{width: "16rem"}}
+              onChange={(_ev, data) => setSearchValue(data.value)} />
             <Tooltip content="Download Data as JSON" relationship="description">
               <Button icon={<ArrowDownloadRegular />} appearance="subtle" onClick={downloadDataset} />
             </Tooltip>

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ReportContext.tsx

@@ -1,5 +1,5 @@
 import { useContext, createContext, useState } from "react";
-import { ScoreNode, ScoreNodeType, ScoreSummary } from "./Summary";
+import { ReverseTextIndex, ScoreNode, ScoreNodeType, ScoreSummary } from "./Summary";
 
 export type ReportContextType = {
     dataset: Dataset,
@@ -8,9 +8,12 @@ export type ReportContextType = {
     selectScenarioLevel: (key: string) => void,
     renderMarkdown: boolean,
     setRenderMarkdown: (renderMarkdown: boolean) => void,
+    searchValue: string,
+    setSearchValue: (searchValue: string) => void,
     selectedTags: string[],
     handleTagClick: (tag: string) => void,
     clearFilters: () => void,
+    filterTree: (node: ScoreNode) => ScoreNode | null,
 };
 
 // Create the default context, which will be used to provide the context value
@@ -22,6 +25,7 @@ const defaultReportContext = createContext<ReportContextType>({
         includesReportHistory: false,
         executionHistory: new Map<string, ScoreNode>(),
         nodesByKey: new Map<string, Map<string, ScoreNode>>(),
+        reverseTextIndex: new ReverseTextIndex(),
     },
     selectedScenarioLevel: undefined,
     selectScenarioLevel: (_selectedScenarioLevel: string) => {
@@ -31,9 +35,12 @@ const defaultReportContext = createContext<ReportContextType>({
     setRenderMarkdown: (_renderMarkdown: boolean) => {
         throw new Error("setRenderMarkdown function not implemented");
     },
+    searchValue: '',
+    setSearchValue: (_searchValue: string | undefined) => { throw new Error("setSearchValue function not implemented"); },
     selectedTags: [],
     handleTagClick: (_tag: string) => { throw new Error("handleTagClick function not implemented"); },
     clearFilters: () => { throw new Error("clearFilters function not implemented"); },
+    filterTree: (_node: ScoreNode) => { throw new Error("filterTree function not implemented"); },
 });
 
 export const ReportContextProvider = ({ dataset, scoreSummary, children }:
@@ -56,6 +63,7 @@ const useProvideReportContext = (dataset: Dataset, scoreSummary: ScoreSummary):
     const [selectedScenarioLevel, setSelectedScenarioLevel] = useState<string | undefined>(undefined);
     const [renderMarkdown, setRenderMarkdown] = useState<boolean>(true);
     const [selectedTags, setSelectedTags] = useState<string[]>([]);
+    const [searchValue, setSearchValue] = useState<string>("");
 
     const selectScenarioLevel = (key: string) => {
         if (key === selectedScenarioLevel) {
@@ -74,17 +82,52 @@ const useProvideReportContext = (dataset: Dataset, scoreSummary: ScoreSummary):
 
     const clearFilters = () => {
         setSelectedTags([]);
+        setSearchValue("");
     };
 
+    const filterTree = (node: ScoreNode): ScoreNode | null => {
+        if (selectedTags.length === 0 && searchValue === "") {
+            return node;
+        }
+
+        const searchedNodes = scoreSummary.reverseTextIndex.search(searchValue);
+
+        const srch = (node: ScoreNode) : ScoreNode | null => {
+            if (node.isLeafNode) {
+                const tagMatches = selectedTags.length > 0 && node.scenario?.tags?.some(tag => selectedTags.includes(tag));
+                const searchMatches = searchValue !== "" && searchedNodes.has(node.nodeKey);
+                return tagMatches || searchMatches ? node : null;
+            }
+
+            const filteredChildren = node.childNodes
+                .map(srch)
+                .filter((child): child is ScoreNode => child !== null);
+
+            if (filteredChildren.length > 0) {
+                const newNode = new ScoreNode(node.name, node.nodeType, node.nodeKey, node.executionName);
+                newNode.setChildren(new Map(filteredChildren.map(child => [child.name, child])));
+                newNode.aggregate();
+                return newNode;
+            }
+
+            return null;
+        };  
+
+        return srch(node);
+    }
+
     return {
         dataset,
         scoreSummary,
         selectedScenarioLevel,
         selectScenarioLevel,
         renderMarkdown,
         setRenderMarkdown,
+        searchValue,
+        setSearchValue,
         selectedTags,
         handleTagClick,
-        clearFilters
+        clearFilters,
+        filterTree,
     };
 };

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ScenarioTree.tsx

@@ -46,36 +46,13 @@ export const ScenarioGroup = ({ node, scoreSummary }: {
     node: ScoreNode,
     scoreSummary: ScoreSummary,
 }) => {
-    const { selectedTags } = useReportContext();
+    const { filterTree } = useReportContext();
     const [openItems, setOpenItems] = useState<Set<TreeItemValue>>(() => new Set());
     const handleOpenChange = useCallback((_: TreeOpenChangeEvent, data: TreeOpenChangeData) => {
         setOpenItems(data.openItems);
     }, []);
     const isOpen = (name: string) => openItems.has(name);
 
-    const filterTree = (node: ScoreNode): ScoreNode | null => {
-        if (selectedTags.length === 0) {
-            return node;
-        }
-
-        if (node.isLeafNode) {
-            return node.scenario?.tags?.some(tag => selectedTags.includes(tag)) ? node : null;
-        }
-
-        const filteredChildren = node.childNodes
-            .map(filterTree)
-            .filter((child): child is ScoreNode => child !== null);
-
-        if (filteredChildren.length > 0) {
-            const newNode = new ScoreNode(node.name, node.nodeType, node.nodeKey, node.executionName);
-            newNode.setChildren(new Map(filteredChildren.map(child => [child.name, child])));
-            newNode.aggregate(selectedTags);
-            return newNode;
-        }
-
-        return null;
-    };
-
     const filteredNode = filterTree(node);
 
     if (!filteredNode) {

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+import uFuzzy from "@leeoniya/ufuzzy";
+
 export enum ScoreNodeType {
     Group,
     Scenario,
@@ -12,6 +14,7 @@ export type ScoreSummary = {
     includesReportHistory: boolean;
     executionHistory: Map<string, ScoreNode>;
     nodesByKey: Map<string, Map<string, ScoreNode>>;
+    reverseTextIndex: ReverseTextIndex
 };
 
 export class ScoreNode {
@@ -80,17 +83,14 @@ export class ScoreNode {
         return [...flattener(this)];
     }
 
-    aggregate(filteredTags: string[] = []) {
+    aggregate() {
         this.failed = false;
         this.numPassingIterations = 0;
         this.numFailingIterations = 0;
         this.numPassingScenarios = 0;
         this.numFailingScenarios = 0;
 
         if (this.isLeafNode) {
-            if (filteredTags.length > 0 && !this.scenario?.tags?.some(tag => filteredTags.includes(tag))) {
-                return;
-            }
 
             this.failed = false;
             for (const metric of Object.values(this.scenario?.evaluationResult.metrics ?? [])) {
@@ -116,8 +116,8 @@ export class ScoreNode {
             this.shortenedPrompt = shortenPrompt(history);
         } else {
             for (const child of this.childNodes) {
-                child.aggregate(filteredTags);
-                if (filteredTags.length === 0 || child.numPassingIterations + child.numFailingIterations > 0) {
+                child.aggregate();
+                if (child.numPassingIterations + child.numFailingIterations > 0) {
                     this.failed = this.failed || child.failed;
                     this.numPassingIterations += child.numPassingIterations;
                     this.numFailingIterations += child.numFailingIterations;
@@ -152,6 +152,38 @@ export class ScoreNode {
     }
 };
 
+export class ReverseTextIndex {
+
+    private stringsToSearch: string[] = [];
+    private keys: string[] = [];
+
+    addText(key: string, text?: string) {
+        if (!text) {
+            return;
+        }
+        this.stringsToSearch.push(text);
+        this.keys.push(key);
+    }
+
+    search(searchValue: string): Set<string> {
+        const opts = {
+            intraMode: 0,
+            unicode: true,
+        } as uFuzzy.Options;
+        const fz = new uFuzzy(opts);
+        const terms = fz.split(searchValue);
+        const keys = new Set<string>();
+        for (const term of terms) {
+            const searchResult = fz.search(this.stringsToSearch, term) as uFuzzy.FilteredResult;
+            const matches = searchResult[0];
+            for (const match of matches) {
+                keys.add(this.keys[match]);
+            }
+        }
+        return keys;
+    }
+}
+
 export const createScoreSummary = (dataset: Dataset): ScoreSummary => {
 
     const executionHistory = new Map<string, ScoreNode>();
@@ -183,11 +215,34 @@ export const createScoreSummary = (dataset: Dataset): ScoreSummary => {
     const [primaryResult] = executionHistory.values();
     primaryResult.collapseSingleChildNodes();
 
+    const reverseTextIndex = new ReverseTextIndex();
+
+    // build the reverse text index from searchable strings in the data
+    for (const node of primaryResult.flattenedNodes) {
+        reverseTextIndex.addText(node.nodeKey, node.scenario?.scenarioName);
+        reverseTextIndex.addText(node.nodeKey, node.scenario?.iterationName);
+        for (const message of node.scenario?.messages ?? []) {
+            for (const content of message.contents) {
+                if (isTextContent(content)) {
+                    reverseTextIndex.addText(node.nodeKey, content.text);
+                }
+            }
+        }
+        for (const message of node.scenario?.modelResponse?.messages ?? []) {
+            for (const content of message.contents) {
+                if (isTextContent(content)) {
+                    reverseTextIndex.addText(node.nodeKey, content.text);
+                }
+            }
+        }
+    }
+
     return {
         primaryResult,
         includesReportHistory: executionHistory.size > 1,
         executionHistory,
         nodesByKey,
+        reverseTextIndex,
     } as ScoreSummary;
 };