🚀 feat: Use Model Specs + Specific Endpoints, Limit Providers for Agents (danny-avila#6650)

* 🔧 refactor: Remove modelSpecs prop from ModelSelector and related components

* fix: Update submission.conversationId references in SSE hooks and data types as was incorrectly typed

* feat: Allow showing specific endpoints alongside model specs via `addedEndpoints` field

* feat: allowed agents providers via `agents.allowedProviders` field

* fix: bump dicebear/sharp dependencies to resolve CVE-2024-12905 and improve avatar gen logic

* fix: rename variable for clarity in loadDefaultInterface function

* fix: add keepAddedConvos option to newConversation calls for modular chat support

* fix: include model information in endpoint selection for improved context

* fix: update data-provider version to 0.7.78 and increment config version to 1.2.4

Author: danny-avila

api/package.json

@@ -104,7 +104,7 @@
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "rate-limit-redis": "^4.2.0",
-    "sharp": "^0.32.6",
+    "sharp": "^0.33.5",
     "tiktoken": "^1.0.15",
     "traverse": "^0.6.7",
     "ua-parser-js": "^1.0.36",

api/server/services/AppService.js

@@ -146,7 +146,7 @@ const AppService = async (app) => {
     ...defaultLocals,
     fileConfig: config?.fileConfig,
     secureImageLinks: config?.secureImageLinks,
-    modelSpecs: processModelSpecs(endpoints, config.modelSpecs),
+    modelSpecs: processModelSpecs(endpoints, config.modelSpecs, interfaceConfig),
     ...endpointLocals,
   };
 };

api/server/services/Config/getEndpointsConfig.js

@@ -33,10 +33,12 @@ async function getEndpointsConfig(req) {
     };
   }
   if (mergedConfig[EModelEndpoint.agents] && req.app.locals?.[EModelEndpoint.agents]) {
-    const { disableBuilder, capabilities, ..._rest } = req.app.locals[EModelEndpoint.agents];
+    const { disableBuilder, capabilities, allowedProviders, ..._rest } =
+      req.app.locals[EModelEndpoint.agents];
 
     mergedConfig[EModelEndpoint.agents] = {
       ...mergedConfig[EModelEndpoint.agents],
+      allowedProviders,
       disableBuilder,
       capabilities,
     };

api/server/services/Endpoints/agents/initialize.js

@@ -1,5 +1,6 @@
 const { createContentAggregator, Providers } = require('@librechat/agents');
 const {
+  ErrorTypes,
   EModelEndpoint,
   getResponseSender,
   AgentCapabilities,
@@ -117,6 +118,7 @@ function optionalChainWithEmptyCheck(...values) {
  * @param {ServerRequest} params.req
  * @param {ServerResponse} params.res
  * @param {Agent} params.agent
+ * @param {Set<string>} [params.allowedProviders]
  * @param {object} [params.endpointOption]
  * @param {boolean} [params.isInitialAgent]
  * @returns {Promise<Agent>}
@@ -126,8 +128,14 @@ const initializeAgentOptions = async ({
   res,
   agent,
   endpointOption,
+  allowedProviders,
   isInitialAgent = false,
 }) => {
+  if (allowedProviders.size > 0 && !allowedProviders.has(agent.provider)) {
+    throw new Error(
+      `{ "type": "${ErrorTypes.INVALID_AGENT_PROVIDER}", "info": "${agent.provider}" }`,
+    );
+  }
   let currentFiles;
   /** @type {Array<MongoFile>} */
   const requestFiles = req.body.files ?? [];
@@ -263,13 +271,16 @@ const initializeClient = async ({ req, res, endpointOption }) => {
   }
 
   const agentConfigs = new Map();
+  /** @type {Set<string>} */
+  const allowedProviders = new Set(req?.app?.locals?.[EModelEndpoint.agents]?.allowedProviders);
 
   // Handle primary agent
   const primaryConfig = await initializeAgentOptions({
     req,
     res,
     agent: primaryAgent,
     endpointOption,
+    allowedProviders,
     isInitialAgent: true,
   });
 
@@ -285,6 +296,7 @@ const initializeClient = async ({ req, res, endpointOption }) => {
         res,
         agent,
         endpointOption,
+        allowedProviders,
       });
       agentConfigs.set(agentId, config);
     }

api/server/services/start/interface.js

@@ -18,12 +18,15 @@ async function loadDefaultInterface(config, configDefaults, roleName = SystemRol
   const { interface: interfaceConfig } = config ?? {};
   const { interface: defaults } = configDefaults;
   const hasModelSpecs = config?.modelSpecs?.list?.length > 0;
+  const includesAddedEndpoints = config?.modelSpecs?.addedEndpoints?.length > 0;
 
   /** @type {TCustomConfig['interface']} */
   const loadedInterface = removeNullishValues({
     endpointsMenu:
       interfaceConfig?.endpointsMenu ?? (hasModelSpecs ? false : defaults.endpointsMenu),
-    modelSelect: interfaceConfig?.modelSelect ?? (hasModelSpecs ? false : defaults.modelSelect),
+    modelSelect:
+      interfaceConfig?.modelSelect ??
+      (hasModelSpecs ? includesAddedEndpoints : defaults.modelSelect),
     parameters: interfaceConfig?.parameters ?? (hasModelSpecs ? false : defaults.parameters),
     presets: interfaceConfig?.presets ?? (hasModelSpecs ? false : defaults.presets),
     sidePanel: interfaceConfig?.sidePanel ?? defaults.sidePanel,

api/server/services/start/modelSpecs.js

@@ -6,9 +6,10 @@ const { logger } = require('~/config');
  * Sets up Model Specs from the config (`librechat.yaml`) file.
  * @param {TCustomConfig['endpoints']} [endpoints] - The loaded custom configuration for endpoints.
  * @param {TCustomConfig['modelSpecs'] | undefined} [modelSpecs] - The loaded custom configuration for model specs.
+ * @param {TCustomConfig['interface'] | undefined} [interfaceConfig] - The loaded interface configuration.
  * @returns {TCustomConfig['modelSpecs'] | undefined} The processed model specs, if any.
  */
-function processModelSpecs(endpoints, _modelSpecs) {
+function processModelSpecs(endpoints, _modelSpecs, interfaceConfig) {
   if (!_modelSpecs) {
     return undefined;
   }
@@ -20,6 +21,19 @@ function processModelSpecs(endpoints, _modelSpecs) {
 
   const customEndpoints = endpoints?.[EModelEndpoint.custom] ?? [];
 
+  if (interfaceConfig.modelSelect !== true && _modelSpecs.addedEndpoints.length > 0) {
+    logger.warn(
+      `To utilize \`addedEndpoints\`, which allows provider/model selections alongside model specs, set \`modelSelect: true\` in the interface configuration.
+
+      Example:
+      \`\`\`yaml
+      interface:
+        modelSelect: true
+      \`\`\`
+      `,
+    );
+  }
+
   for (const spec of list) {
     if (EModelEndpoint[spec.preset.endpoint] && spec.preset.endpoint !== EModelEndpoint.custom) {
       modelSpecs.push(spec);

client/package.json

@@ -31,8 +31,8 @@
     "@ariakit/react": "^0.4.15",
     "@ariakit/react-core": "^0.4.15",
     "@codesandbox/sandpack-react": "^2.19.10",
-    "@dicebear/collection": "^7.0.4",
-    "@dicebear/core": "^7.0.4",
+    "@dicebear/collection": "^9.2.2",
+    "@dicebear/core": "^9.2.2",
     "@headlessui/react": "^2.1.2",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.2",

client/src/common/selector.ts

@@ -20,5 +20,4 @@ export interface SelectedValues {
 
 export interface ModelSelectorProps {
   startupConfig: TStartupConfig | undefined;
-  modelSpecs: TModelSpec[];
 }

client/src/components/Chat/Header.tsx

@@ -16,7 +16,6 @@ const defaultInterface = getConfigDefaults().interface;
 export default function Header() {
   const { data: startupConfig } = useGetStartupConfig();
   const { navVisible } = useOutletContext<ContextType>();
-  const modelSpecs = useMemo(() => startupConfig?.modelSpecs?.list ?? [], [startupConfig]);
   const interfaceConfig = useMemo(
     () => startupConfig?.interface ?? defaultInterface,
     [startupConfig],
@@ -39,7 +38,7 @@ export default function Header() {
       <div className="hide-scrollbar flex w-full items-center justify-between gap-2 overflow-x-auto">
         <div className="mx-2 flex items-center gap-2">
           {!navVisible && <HeaderNewChat />}
-          {<ModelSelector startupConfig={startupConfig} modelSpecs={modelSpecs} />}
+          {<ModelSelector startupConfig={startupConfig} />}
           {interfaceConfig.presets === true && interfaceConfig.modelSelect && <PresetsMenu />}
           {hasAccessToBookmarks === true && <BookmarkMenu />}
           {hasAccessToMultiConvo === true && <AddMultiConvo />}

client/src/components/Chat/Menus/Endpoints/ModelSelector.tsx

@@ -98,9 +98,9 @@ function ModelSelectorContent() {
   );
 }
 
-export default function ModelSelector({ startupConfig, modelSpecs }: ModelSelectorProps) {
+export default function ModelSelector({ startupConfig }: ModelSelectorProps) {
   return (
-    <ModelSelectorProvider modelSpecs={modelSpecs} startupConfig={startupConfig}>
+    <ModelSelectorProvider startupConfig={startupConfig}>
       <ModelSelectorContent />
     </ModelSelectorProvider>
   );