refactor(plugins): Improve OpenAPI handling, Show Multiple Plugins, & Other Improvements

api/app/clients/PluginsClient.js

@@ -1,12 +1,10 @@
 const OpenAIClient = require('./OpenAIClient');
-const { ChatOpenAI } = require('langchain/chat_models/openai');
 const { CallbackManager } = require('langchain/callbacks');
+const { HumanChatMessage, AIChatMessage } = require('langchain/schema');
 const { initializeCustomAgent, initializeFunctionsAgent } = require('./agents/');
-const { findMessageContent } = require('../../utils');
-const { loadTools } = require('./tools/util');
+const { addImages, createLLM, buildErrorInput, buildPromptPrefix } = require('./agents/methods/');
 const { SelfReflectionTool } = require('./tools/');
-const { HumanChatMessage, AIChatMessage } = require('langchain/schema');
-const { instructions, imageInstructions, errorInstructions } = require('./prompts/instructions');
+const { loadTools } = require('./tools/util');
 
 class PluginsClient extends OpenAIClient {
   constructor(apiKey, options = {}) {
@@ -19,89 +17,6 @@ class PluginsClient extends OpenAIClient {
     this.executor = null;
   }
 
-  getActions(input = null) {
-    let output = 'Internal thoughts & actions taken:\n"';
-    let actions = input || this.actions;
-
-    if (actions[0]?.action && this.functionsAgent) {
-      actions = actions.map((step) => ({
-        log: `Action: ${step.action?.tool || ''}\nInput: ${
-          JSON.stringify(step.action?.toolInput) || ''
-        }\nObservation: ${step.observation}`,
-      }));
-    } else if (actions[0]?.action) {
-      actions = actions.map((step) => ({
-        log: `${step.action.log}\nObservation: ${step.observation}`,
-      }));
-    }
-
-    actions.forEach((actionObj, index) => {
-      output += `${actionObj.log}`;
-      if (index < actions.length - 1) {
-        output += '\n';
-      }
-    });
-
-    return output + '"';
-  }
-
-  buildErrorInput(message, errorMessage) {
-    const log = errorMessage.includes('Could not parse LLM output:')
-      ? `A formatting error occurred with your response to the human's last message. You didn't follow the formatting instructions. Remember to ${instructions}`
-      : `You encountered an error while replying to the human's last message. Attempt to answer again or admit an answer cannot be given.\nError: ${errorMessage}`;
-
-    return `
-      ${log}
-
-      ${this.getActions()}
-
-      Human's last message: ${message}
-      `;
-  }
-
-  buildPromptPrefix(result, message) {
-    if ((result.output && result.output.includes('N/A')) || result.output === undefined) {
-      return null;
-    }
-
-    if (
-      result?.intermediateSteps?.length === 1 &&
-      result?.intermediateSteps[0]?.action?.toolInput === 'N/A'
-    ) {
-      return null;
-    }
-
-    const internalActions =
-      result?.intermediateSteps?.length > 0
-        ? this.getActions(result.intermediateSteps)
-        : 'Internal Actions Taken: None';
-
-    const toolBasedInstructions = internalActions.toLowerCase().includes('image')
-      ? imageInstructions
-      : '';
-
-    const errorMessage = result.errorMessage ? `${errorInstructions} ${result.errorMessage}\n` : '';
-
-    const preliminaryAnswer =
-      result.output?.length > 0 ? `Preliminary Answer: "${result.output.trim()}"` : '';
-    const prefix = preliminaryAnswer
-      ? 'review and improve the answer you generated using plugins in response to the User Message below. The user hasn\'t seen your answer or thoughts yet.'
-      : 'respond to the User Message below based on your preliminary thoughts & actions.';
-
-    return `As a helpful AI Assistant, ${prefix}${errorMessage}\n${internalActions}
-${preliminaryAnswer}
-Reply conversationally to the User based on your ${
-  preliminaryAnswer ? 'preliminary answer, ' : ''
-}internal actions, thoughts, and observations, making improvements wherever possible, but do not modify URLs.
-${
-  preliminaryAnswer
-    ? ''
-    : '\nIf there is an incomplete thought or action, you are expected to complete it in your response now.\n'
-}You must cite sources if you are using any web links. ${toolBasedInstructions}
-Only respond with your conversational reply to the following User Message:
-"${message}"`;
-  }
-
   setOptions(options) {
     this.agentOptions = options.agentOptions;
     this.functionsAgent = this.agentOptions?.agent === 'functions';
@@ -149,27 +64,6 @@ Only respond with your conversational reply to the following User Message:
     };
   }
 
-  createLLM(modelOptions, configOptions) {
-    let azure = {};
-    let credentials = { openAIApiKey: this.openAIApiKey };
-    let configuration = {
-      apiKey: this.openAIApiKey,
-    };
-
-    if (this.azure) {
-      credentials = {};
-      configuration = {};
-      ({ azure } = this);
-    }
-
-    if (this.options.debug) {
-      console.debug('createLLM: configOptions');
-      console.debug(configOptions);
-    }
-
-    return new ChatOpenAI({ credentials, configuration, ...azure, ...modelOptions }, configOptions);
-  }
-
   async initialize({ user, message, onAgentAction, onChainEnd, signal }) {
     const modelOptions = {
       modelName: this.agentOptions.model,
@@ -182,35 +76,36 @@ Only respond with your conversational reply to the following User Message:
       configOptions.basePath = this.langchainProxy;
     }
 
-    const model = this.createLLM(modelOptions, configOptions);
+    const model = createLLM({
+      modelOptions,
+      configOptions,
+      openAIApiKey: this.openAIApiKey,
+      azure: this.azure,
+    });
 
     if (this.options.debug) {
       console.debug(
         `<-----Agent Model: ${model.modelName} | Temp: ${model.temperature} | Functions: ${this.functionsAgent}----->`,
       );
     }
 
-    this.availableTools = await loadTools({
+    this.tools = await loadTools({
       user,
       model,
       tools: this.options.tools,
       functions: this.functionsAgent,
       options: {
         openAIApiKey: this.openAIApiKey,
+        conversationId: this.conversationId,
         debug: this.options?.debug,
         message,
       },
     });
-    // load tools
-    for (const tool of this.options.tools) {
-      const validTool = this.availableTools[tool];
-
-      if (tool === 'plugins') {
-        const plugins = await validTool();
-        this.tools = [...this.tools, ...plugins];
-      } else if (validTool) {
-        this.tools.push(await validTool());
-      }
+
+    if (this.tools.length > 0 && !this.functionsAgent) {
+      this.tools.push(new SelfReflectionTool({ message, isGpt3: false }));
+    } else if (this.tools.length === 0) {
+      return;
     }
 
     if (this.options.debug) {
@@ -220,21 +115,15 @@ Only respond with your conversational reply to the following User Message:
       console.debug(this.tools.map((tool) => tool.name));
     }
 
-    if (this.tools.length > 0 && !this.functionsAgent) {
-      this.tools.push(new SelfReflectionTool({ message, isGpt3: false }));
-    } else if (this.tools.length === 0) {
-      return;
-    }
-
-    const handleAction = (action, callback = null) => {
+    const handleAction = (action, runId, callback = null) => {
       this.saveLatestAction(action);
 
       if (this.options.debug) {
         console.debug('Latest Agent Action ', this.actions[this.actions.length - 1]);
       }
 
       if (typeof callback === 'function') {
-        callback(action);
+        callback(action, runId);
       }
     };
 
@@ -258,8 +147,8 @@ Only respond with your conversational reply to the following User Message:
       verbose: this.options.debug,
       returnIntermediateSteps: true,
       callbackManager: CallbackManager.fromHandlers({
-        async handleAgentAction(action) {
-          handleAction(action, onAgentAction);
+        async handleAgentAction(action, runId) {
+          handleAction(action, runId, onAgentAction);
         },
         async handleChainEnd(action) {
           if (typeof onChainEnd === 'function') {
@@ -274,12 +163,17 @@ Only respond with your conversational reply to the following User Message:
     }
   }
 
-  async executorCall(message, signal) {
+  async executorCall(message, { signal, stream, onToolStart, onToolEnd }) {
     let errorMessage = '';
     const maxAttempts = 1;
 
     for (let attempts = 1; attempts <= maxAttempts; attempts++) {
-      const errorInput = this.buildErrorInput(message, errorMessage);
+      const errorInput = buildErrorInput({
+        message,
+        errorMessage,
+        actions: this.actions,
+        functionsAgent: this.functionsAgent,
+      });
       const input = attempts > 1 ? errorInput : message;
 
       if (this.options.debug) {
@@ -291,12 +185,28 @@ Only respond with your conversational reply to the following User Message:
       }
 
       try {
-        this.result = await this.executor.call({ input, signal });
+        this.result = await this.executor.call({ input, signal }, [
+          {
+            async handleToolStart(...args) {
+              await onToolStart(...args);
+            },
+            async handleToolEnd(...args) {
+              await onToolEnd(...args);
+            },
+            async handleLLMEnd(output) {
+              const { generations } = output;
+              const { text } = generations[0][0];
+              if (text && typeof stream === 'function') {
+                await stream(text);
+              }
+            },
+          },
+        ]);
         break; // Exit the loop if the function call is successful
       } catch (err) {
         console.error(err);
         errorMessage = err.message;
-        const content = findMessageContent(message);
+        let content = '';
         if (content) {
           errorMessage = content;
           break;
@@ -311,31 +221,6 @@ Only respond with your conversational reply to the following User Message:
     }
   }
 
-  addImages(intermediateSteps, responseMessage) {
-    if (!intermediateSteps || !responseMessage) {
-      return;
-    }
-
-    intermediateSteps.forEach((step) => {
-      const { observation } = step;
-      if (!observation || !observation.includes('![')) {
-        return;
-      }
-
-      // Extract the image file path from the observation
-      const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g)[0];
-
-      // Check if the responseMessage already includes the image file path
-      if (!responseMessage.text.includes(observedImagePath)) {
-        // If the image file path is not found, append the whole observation
-        responseMessage.text += '\n' + observation;
-        if (this.options.debug) {
-          console.debug('added image from intermediateSteps');
-        }
-      }
-    });
-  }
-
   async handleResponseMessage(responseMessage, saveOptions, user) {
     responseMessage.tokenCount = this.getTokenCountForResponse(responseMessage);
     responseMessage.completionTokens = responseMessage.tokenCount;
@@ -351,7 +236,9 @@ Only respond with your conversational reply to the following User Message:
       this.setOptions(opts);
       return super.sendMessage(message, opts);
     }
-    console.log('Plugins sendMessage', message, opts);
+    if (this.options.debug) {
+      console.log('Plugins sendMessage', message, opts);
+    }
     const {
       user,
       conversationId,
@@ -360,8 +247,11 @@ Only respond with your conversational reply to the following User Message:
       userMessage,
       onAgentAction,
       onChainEnd,
+      onToolStart,
+      onToolEnd,
     } = await this.handleStartMethods(message, opts);
 
+    this.conversationId = conversationId;
     this.currentMessages.push(userMessage);
 
     let {
@@ -413,19 +303,38 @@ Only respond with your conversational reply to the following User Message:
       onAgentAction,
       onChainEnd,
       signal: this.abortController.signal,
+      onProgress: opts.onProgress,
+    });
+
+    // const stream = async (text) => {
+    //   await this.generateTextStream.call(this, text, opts.onProgress, { delay: 1 });
+    // };
+    await this.executorCall(message, {
+      signal: this.abortController.signal,
+      // stream,
+      onToolStart,
+      onToolEnd,
     });
-    await this.executorCall(message, this.abortController.signal);
 
     // If message was aborted mid-generation
     if (this.result?.errorMessage?.length > 0 && this.result?.errorMessage?.includes('cancel')) {
       responseMessage.text = 'Cancelled.';
       return await this.handleResponseMessage(responseMessage, saveOptions, user);
     }
 
+    if (this.agentOptions.skipCompletion && this.result.output && this.functionsAgent) {
+      const partialText = opts.getPartialText();
+      const trimmedPartial = opts.getPartialText().replaceAll(':::plugin:::\n', '');
+      responseMessage.text =
+        trimmedPartial.length === 0 ? `${partialText}${this.result.output}` : partialText;
+      await this.generateTextStream(this.result.output, opts.onProgress, { delay: 5 });
+      return await this.handleResponseMessage(responseMessage, saveOptions, user);
+    }
+
     if (this.agentOptions.skipCompletion && this.result.output) {
       responseMessage.text = this.result.output;
-      this.addImages(this.result.intermediateSteps, responseMessage);
-      await this.generateTextStream(this.result.output, opts.onProgress, { delay: 8 });
+      addImages(this.result.intermediateSteps, responseMessage);
+      await this.generateTextStream(this.result.output, opts.onProgress, { delay: 5 });
       return await this.handleResponseMessage(responseMessage, saveOptions, user);
     }
 
@@ -434,7 +343,11 @@ Only respond with your conversational reply to the following User Message:
       console.debug(this.result);
     }
 
-    const promptPrefix = this.buildPromptPrefix(this.result, message);
+    const promptPrefix = buildPromptPrefix({
+      result: this.result,
+      message,
+      functionsAgent: this.functionsAgent,
+    });
 
     if (this.options.debug) {
       console.debug('Plugins: promptPrefix');