Skip to content

Commit 48c087c

Browse files
danny-aviladanny-avila
authored
chore: add token rate support for 11/06 models (danny-avila#1146)
* chore: update model rates with 11/06 rates * chore: add new models to env.example for OPENAI_MODELS * chore: reference actual maxTokensMap in ci tests
1 parent 4b63eb5 commit 48c087c

File tree

5 files changed

+110
-11
lines changed

5 files changed

+110
-11
lines changed

‎.env.example

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ DEBUG_OPENAI=false # Set to true to enable debug mode for the OpenAI endpoint
9797
# Identify the available models, separated by commas *without spaces*.
9898
# The first will be default.
9999
# Leave it blank to use internal settings.
100-
# OPENAI_MODELS=gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,text-davinci-003,gpt-4,gpt-4-0314,gpt-4-0613
100+
# OPENAI_MODELS=gpt-3.5-turbo-1106,gpt-4-1106-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-0301,text-davinci-003,gpt-4,gpt-4-0314,gpt-4-0613
101101

102102
# Titling is enabled by default when initiating a conversation.
103103
# Uncomment the following variable to disable this feature.

‎api/models/tx.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ const tokenValues = {
1010
'32k': { prompt: 60, completion: 120 },
1111
'4k': { prompt: 1.5, completion: 2 },
1212
'16k': { prompt: 3, completion: 4 },
13+
'gpt-3.5-turbo-1106': { prompt: 1, completion: 2 },
14+
'gpt-4-1106': { prompt: 10, completion: 30 },
1315
};
1416

1517
/**
@@ -26,8 +28,12 @@ const getValueKey = (model) => {
2628

2729
if (modelName.includes('gpt-3.5-turbo-16k')) {
2830
return '16k';
31+
} else if (modelName.includes('gpt-3.5-turbo-1106')) {
32+
return 'gpt-3.5-turbo-1106';
2933
} else if (modelName.includes('gpt-3.5')) {
3034
return '4k';
35+
} else if (modelName.includes('gpt-4-1106')) {
36+
return 'gpt-4-1106';
3137
} else if (modelName.includes('gpt-4-32k')) {
3238
return '32k';
3339
} else if (modelName.includes('gpt-4')) {

‎api/models/tx.spec.js

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const { getValueKey, getMultiplier, defaultRate } = require('./tx');
1+
const { getValueKey, getMultiplier, defaultRate, tokenValues } = require('./tx');
22

33
describe('getValueKey', () => {
44
it('should return "16k" for model name containing "gpt-3.5-turbo-16k"', () => {
@@ -20,27 +20,72 @@ describe('getValueKey', () => {
2020
it('should return undefined for model names that do not match any known patterns', () => {
2121
expect(getValueKey('gpt-5-some-other-info')).toBeUndefined();
2222
});
23+
24+
it('should return "gpt-3.5-turbo-1106" for model name containing "gpt-3.5-turbo-1106"', () => {
25+
expect(getValueKey('gpt-3.5-turbo-1106-some-other-info')).toBe('gpt-3.5-turbo-1106');
26+
expect(getValueKey('openai/gpt-3.5-turbo-1106')).toBe('gpt-3.5-turbo-1106');
27+
expect(getValueKey('gpt-3.5-turbo-1106/openai')).toBe('gpt-3.5-turbo-1106');
28+
});
29+
30+
it('should return "gpt-4-1106" for model name containing "gpt-4-1106"', () => {
31+
expect(getValueKey('gpt-4-1106-some-other-info')).toBe('gpt-4-1106');
32+
expect(getValueKey('gpt-4-1106-vision-preview')).toBe('gpt-4-1106');
33+
expect(getValueKey('gpt-4-1106-preview')).toBe('gpt-4-1106');
34+
expect(getValueKey('openai/gpt-4-1106')).toBe('gpt-4-1106');
35+
expect(getValueKey('gpt-4-1106/openai/')).toBe('gpt-4-1106');
36+
});
2337
});
2438

2539
describe('getMultiplier', () => {
2640
it('should return the correct multiplier for a given valueKey and tokenType', () => {
27-
expect(getMultiplier({ valueKey: '8k', tokenType: 'prompt' })).toBe(30);
28-
expect(getMultiplier({ valueKey: '8k', tokenType: 'completion' })).toBe(60);
41+
expect(getMultiplier({ valueKey: '8k', tokenType: 'prompt' })).toBe(tokenValues['8k'].prompt);
42+
expect(getMultiplier({ valueKey: '8k', tokenType: 'completion' })).toBe(
43+
tokenValues['8k'].completion,
44+
);
2945
});
3046

3147
it('should return defaultRate if tokenType is provided but not found in tokenValues', () => {
3248
expect(getMultiplier({ valueKey: '8k', tokenType: 'unknownType' })).toBe(defaultRate);
3349
});
3450

3551
it('should derive the valueKey from the model if not provided', () => {
36-
expect(getMultiplier({ tokenType: 'prompt', model: 'gpt-4-some-other-info' })).toBe(30);
52+
expect(getMultiplier({ tokenType: 'prompt', model: 'gpt-4-some-other-info' })).toBe(
53+
tokenValues['8k'].prompt,
54+
);
3755
});
3856

3957
it('should return 1 if only model or tokenType is missing', () => {
4058
expect(getMultiplier({ tokenType: 'prompt' })).toBe(1);
4159
expect(getMultiplier({ model: 'gpt-4-some-other-info' })).toBe(1);
4260
});
4361

62+
it('should return the correct multiplier for gpt-3.5-turbo-1106', () => {
63+
expect(getMultiplier({ valueKey: 'gpt-3.5-turbo-1106', tokenType: 'prompt' })).toBe(
64+
tokenValues['gpt-3.5-turbo-1106'].prompt,
65+
);
66+
expect(getMultiplier({ valueKey: 'gpt-3.5-turbo-1106', tokenType: 'completion' })).toBe(
67+
tokenValues['gpt-3.5-turbo-1106'].completion,
68+
);
69+
});
70+
71+
it('should return the correct multiplier for gpt-4-1106', () => {
72+
expect(getMultiplier({ valueKey: 'gpt-4-1106', tokenType: 'prompt' })).toBe(
73+
tokenValues['gpt-4-1106'].prompt,
74+
);
75+
expect(getMultiplier({ valueKey: 'gpt-4-1106', tokenType: 'completion' })).toBe(
76+
tokenValues['gpt-4-1106'].completion,
77+
);
78+
});
79+
80+
it('should derive the valueKey from the model if not provided for new models', () => {
81+
expect(
82+
getMultiplier({ tokenType: 'prompt', model: 'gpt-3.5-turbo-1106-some-other-info' }),
83+
).toBe(tokenValues['gpt-3.5-turbo-1106'].prompt);
84+
expect(getMultiplier({ tokenType: 'completion', model: 'gpt-4-1106-vision-preview' })).toBe(
85+
tokenValues['gpt-4-1106'].completion,
86+
);
87+
});
88+
4489
it('should return defaultRate if derived valueKey does not match any known patterns', () => {
4590
expect(getMultiplier({ tokenType: 'prompt', model: 'gpt-5-some-other-info' })).toBe(
4691
defaultRate,

‎api/utils/tokens.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ const maxTokensMap = {
4949
'gpt-3.5-turbo-0301': 4095,
5050
'gpt-3.5-turbo-16k': 15999,
5151
'gpt-3.5-turbo-16k-0613': 15999,
52+
'gpt-3.5-turbo-1106': 16380, // -5 from max
53+
'gpt-4-1106': 127995, // -5 from max
5254
};
5355

5456
/**

‎api/utils/tokens.spec.js

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,30 @@
1-
const { getModelMaxTokens, matchModelName } = require('./tokens');
1+
const { getModelMaxTokens, matchModelName, maxTokensMap } = require('./tokens');
22

33
describe('getModelMaxTokens', () => {
44
test('should return correct tokens for exact match', () => {
5-
expect(getModelMaxTokens('gpt-4-32k-0613')).toBe(32767);
5+
expect(getModelMaxTokens('gpt-4-32k-0613')).toBe(maxTokensMap['gpt-4-32k-0613']);
66
});
77

88
test('should return correct tokens for partial match', () => {
9-
expect(getModelMaxTokens('gpt-4-32k-unknown')).toBe(32767);
9+
expect(getModelMaxTokens('gpt-4-32k-unknown')).toBe(maxTokensMap['gpt-4-32k']);
1010
});
1111

1212
test('should return correct tokens for partial match (OpenRouter)', () => {
13-
expect(getModelMaxTokens('openai/gpt-4-32k')).toBe(32767);
13+
expect(getModelMaxTokens('openai/gpt-4-32k')).toBe(maxTokensMap['gpt-4-32k']);
1414
});
1515

1616
test('should return undefined for no match', () => {
1717
expect(getModelMaxTokens('unknown-model')).toBeUndefined();
1818
});
1919

2020
test('should return correct tokens for another exact match', () => {
21-
expect(getModelMaxTokens('gpt-3.5-turbo-16k-0613')).toBe(15999);
21+
expect(getModelMaxTokens('gpt-3.5-turbo-16k-0613')).toBe(
22+
maxTokensMap['gpt-3.5-turbo-16k-0613'],
23+
);
2224
});
2325

2426
test('should return correct tokens for another partial match', () => {
25-
expect(getModelMaxTokens('gpt-3.5-turbo-unknown')).toBe(4095);
27+
expect(getModelMaxTokens('gpt-3.5-turbo-unknown')).toBe(maxTokensMap['gpt-3.5-turbo']);
2628
});
2729

2830
test('should return undefined for undefined input', () => {
@@ -36,6 +38,30 @@ describe('getModelMaxTokens', () => {
3638
test('should return undefined for number input', () => {
3739
expect(getModelMaxTokens(123)).toBeUndefined();
3840
});
41+
42+
// 11/06 Update
43+
test('should return correct tokens for gpt-3.5-turbo-1106 exact match', () => {
44+
expect(getModelMaxTokens('gpt-3.5-turbo-1106')).toBe(maxTokensMap['gpt-3.5-turbo-1106']);
45+
});
46+
47+
test('should return correct tokens for gpt-4-1106 exact match', () => {
48+
expect(getModelMaxTokens('gpt-4-1106')).toBe(maxTokensMap['gpt-4-1106']);
49+
});
50+
51+
test('should return correct tokens for gpt-3.5-turbo-1106 partial match', () => {
52+
expect(getModelMaxTokens('something-/gpt-3.5-turbo-1106')).toBe(
53+
maxTokensMap['gpt-3.5-turbo-1106'],
54+
);
55+
expect(getModelMaxTokens('gpt-3.5-turbo-1106/something-/')).toBe(
56+
maxTokensMap['gpt-3.5-turbo-1106'],
57+
);
58+
});
59+
60+
test('should return correct tokens for gpt-4-1106 partial match', () => {
61+
expect(getModelMaxTokens('gpt-4-1106/something')).toBe(maxTokensMap['gpt-4-1106']);
62+
expect(getModelMaxTokens('gpt-4-1106-preview')).toBe(maxTokensMap['gpt-4-1106']);
63+
expect(getModelMaxTokens('gpt-4-1106-vision-preview')).toBe(maxTokensMap['gpt-4-1106']);
64+
});
3965
});
4066

4167
describe('matchModelName', () => {
@@ -57,4 +83,24 @@ describe('matchModelName', () => {
5783
expect(matchModelName(123)).toBeUndefined();
5884
expect(matchModelName({})).toBeUndefined();
5985
});
86+
87+
// 11/06 Update
88+
it('should return the exact model name for gpt-3.5-turbo-1106 if it exists in maxTokensMap', () => {
89+
expect(matchModelName('gpt-3.5-turbo-1106')).toBe('gpt-3.5-turbo-1106');
90+
});
91+
92+
it('should return the exact model name for gpt-4-1106 if it exists in maxTokensMap', () => {
93+
expect(matchModelName('gpt-4-1106')).toBe('gpt-4-1106');
94+
});
95+
96+
it('should return the closest matching key for gpt-3.5-turbo-1106 partial matches', () => {
97+
expect(matchModelName('gpt-3.5-turbo-1106/something')).toBe('gpt-3.5-turbo-1106');
98+
expect(matchModelName('something/gpt-3.5-turbo-1106')).toBe('gpt-3.5-turbo-1106');
99+
});
100+
101+
it('should return the closest matching key for gpt-4-1106 partial matches', () => {
102+
expect(matchModelName('something/gpt-4-1106')).toBe('gpt-4-1106');
103+
expect(matchModelName('gpt-4-1106-preview')).toBe('gpt-4-1106');
104+
expect(matchModelName('gpt-4-1106-vision-preview')).toBe('gpt-4-1106');
105+
});
60106
});

0 commit comments

Comments
 (0)