support responses endpoints as part of unified routes

eslint.config.js

@@ -12,5 +12,6 @@ export default tseslint.config({
   rules: {
     // '@typescript-eslint/no-unused-vars': 'error',
     'no-duplicate-imports': 'error',
+    '@typescript-eslint/no-namespace': 'off',
   },
 });

src/handlers/modelResponsesHandler.ts

@@ -0,0 +1,47 @@
+import { Context } from 'hono';
+import {
+  constructConfigFromRequestHeaders,
+  tryTargetsRecursively,
+} from './handlerUtils';
+import { endpointStrings } from '../providers/types';
+
+function modelResponsesHandler(
+  endpoint: endpointStrings,
+  method: 'POST' | 'GET' | 'DELETE'
+) {
+  async function handler(c: Context): Promise<Response> {
+    try {
+      let requestHeaders = Object.fromEntries(c.req.raw.headers);
+      let request = method === 'POST' ? await c.req.json() : {};
+      const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders);
+      const tryTargetsResponse = await tryTargetsRecursively(
+        c,
+        camelCaseConfig ?? {},
+        request,
+        requestHeaders,
+        endpoint,
+        method,
+        'config'
+      );
+
+      return tryTargetsResponse;
+    } catch (err: any) {
+      console.error({ message: `${endpoint} error ${err.message}` });
+      return new Response(
+        JSON.stringify({
+          status: 'failure',
+          message: 'Something went wrong',
+        }),
+        {
+          status: 500,
+          headers: {
+            'content-type': 'application/json',
+          },
+        }
+      );
+    }
+  }
+  return handler;
+}
+
+export default modelResponsesHandler;

src/handlers/responseHandlers.ts

@@ -16,6 +16,7 @@ import {
 } from './streamHandler';
 import { HookSpan } from '../middlewares/hooks';
 import { env } from 'hono/adapter';
+import { OpenAIModelResponseJSONToStreamGenerator } from '../providers/open-ai-base/createModelResponse';
 
 /**
  * Handles various types of responses based on the specified parameters
@@ -74,10 +75,19 @@ export async function responseHandler(
   // JSON to text/event-stream conversion is only allowed for unified routes: chat completions and completions.
   // Set the transformer to OpenAI json to stream convertor function in that case.
   if (responseTransformer && streamingMode && isCacheHit) {
-    responseTransformerFunction =
-      responseTransformer === 'chatComplete'
-        ? OpenAIChatCompleteJSONToStreamResponseTransform
-        : OpenAICompleteJSONToStreamResponseTransform;
+    switch (responseTransformer) {
+      case 'chatComplete':
+        responseTransformerFunction =
+          OpenAIChatCompleteJSONToStreamResponseTransform;
+        break;
+      case 'createModelResponse':
+        responseTransformerFunction = OpenAIModelResponseJSONToStreamGenerator;
+        break;
+      default:
+        responseTransformerFunction =
+          OpenAICompleteJSONToStreamResponseTransform;
+        break;
+    }
   } else if (responseTransformer && !streamingMode && isCacheHit) {
     responseTransformerFunction = undefined;
   }

src/handlers/streamHandler.ts

@@ -361,14 +361,34 @@ export async function handleJSONToStreamResponse(
   const encoder = new TextEncoder();
   const responseJSON: OpenAIChatCompleteResponse | OpenAICompleteResponse =
     await response.clone().json();
-  const streamChunkArray = responseTransformerFunction(responseJSON, provider);
 
-  (async () => {
-    for (const chunk of streamChunkArray) {
-      await writer.write(encoder.encode(chunk));
-    }
-    writer.close();
-  })();
+  if (
+    Object.prototype.toString.call(responseTransformerFunction) ===
+    '[object GeneratorFunction]'
+  ) {
+    const generator = responseTransformerFunction(responseJSON, provider);
+    (async () => {
+      while (true) {
+        const chunk = generator.next();
+        if (chunk.done) {
+          break;
+        }
+        await writer.write(encoder.encode(chunk.value));
+      }
+      writer.close();
+    })();
+  } else {
+    const streamChunkArray = responseTransformerFunction(
+      responseJSON,
+      provider
+    );
+    (async () => {
+      for (const chunk of streamChunkArray) {
+        await writer.write(encoder.encode(chunk));
+      }
+      writer.close();
+    })();
+  }
 
   return new Response(readable, {
     headers: new Headers({

src/index.ts

@@ -34,6 +34,7 @@ import finetuneHandler from './handlers/finetuneHandler';
 
 // Config
 import conf from '../conf.json';
+import modelResponsesHandler from './handlers/modelResponsesHandler';
 
 // Create a new Hono server instance
 const app = new Hono();
@@ -199,6 +200,28 @@ app.post(
 );
 app.get('/v1/batches', requestValidator, batchesHandler('listBatches', 'GET'));
 
+// responses
+app.post(
+  '/v1/responses',
+  requestValidator,
+  modelResponsesHandler('createModelResponse', 'POST')
+);
+app.get(
+  '/v1/responses/:id',
+  requestValidator,
+  modelResponsesHandler('getModelResponse', 'GET')
+);
+app.delete(
+  '/v1/responses/:id',
+  requestValidator,
+  modelResponsesHandler('deleteModelResponse', 'DELETE')
+);
+app.get(
+  '/v1/responses/:id/input_items',
+  requestValidator,
+  modelResponsesHandler('listResponseInputItems', 'GET')
+);
+
 app.all(
   '/v1/fine_tuning/jobs/:jobId?/:cancel?',
   requestValidator,

src/providers/azure-openai/api.ts

@@ -77,6 +77,12 @@ const AzureOpenAIAPIConfig: ProviderAPIConfig = {
     }
 
     const path = gatewayRequestURL.split('/v1')?.[1];
+    const urlObj = new URL(gatewayRequestURL);
+    const pathname = urlObj.pathname;
+    const searchParams = urlObj.searchParams;
+    if (apiVersion) {
+      searchParams.set('api-version', apiVersion);
+    }
 
     switch (mappedFn) {
       case 'complete': {
@@ -103,6 +109,18 @@ const AzureOpenAIAPIConfig: ProviderAPIConfig = {
       case 'realtime': {
         return `/realtime?api-version=${apiVersion}&deployment=${deploymentId}`;
       }
+      case 'createModelResponse': {
+        return `/responses?${searchParams.toString()}`;
+      }
+      case 'getModelResponse': {
+        return `${pathname}?${searchParams.toString()}`;
+      }
+      case 'deleteModelResponse': {
+        return `${pathname}?${searchParams.toString()}`;
+      }
+      case 'listResponseInputItems': {
+        return `${pathname}?${searchParams.toString()}`;
+      }
       case 'uploadFile':
         return `${path}?api-version=${apiVersion}`;
       case 'retrieveFile':

src/providers/azure-openai/index.ts

@@ -28,6 +28,14 @@ import { OpenAIFileUploadRequestTransform } from '../openai/uploadFile';
 import { AzureOpenAIFinetuneResponseTransform } from './utils';
 import { AzureOpenAICreateBatchConfig } from './createBatch';
 import { AzureOpenAIGetBatchOutputRequestHandler } from './getBatchOutput';
+import {
+  createModelResponseParams,
+  OpenAICreateModelResponseTransformer,
+  OpenAIDeleteModelResponseTransformer,
+  OpenAIGetModelResponseTransformer,
+  OpenAIListInputItemsResponseTransformer,
+} from '../open-ai-base';
+import { AZURE_OPEN_AI } from '../../globals';
 
 const AzureOpenAIConfig: ProviderConfigs = {
   complete: AzureOpenAICompleteConfig,
@@ -43,6 +51,10 @@ const AzureOpenAIConfig: ProviderConfigs = {
   cancelFinetune: {},
   cancelBatch: {},
   createBatch: AzureOpenAICreateBatchConfig,
+  createModelResponse: createModelResponseParams([]),
+  getModelResponse: {},
+  deleteModelResponse: {},
+  listModelsResponse: {},
   requestHandlers: {
     getBatchOutput: AzureOpenAIGetBatchOutputRequestHandler,
   },
@@ -66,6 +78,10 @@ const AzureOpenAIConfig: ProviderConfigs = {
     retrieveBatch: AzureOpenAIResponseTransform,
     cancelBatch: AzureOpenAIResponseTransform,
     listBatches: AzureOpenAIResponseTransform,
+    createModelResponse: OpenAICreateModelResponseTransformer(AZURE_OPEN_AI),
+    getModelResponse: OpenAIGetModelResponseTransformer(AZURE_OPEN_AI),
+    deleteModelResponse: OpenAIDeleteModelResponseTransformer(AZURE_OPEN_AI),
+    listModelsResponse: OpenAIListInputItemsResponseTransformer(AZURE_OPEN_AI),
   },
   requestTransforms: {
     createFinetune: AzureTransformFinetuneBody,