Skip to content

Commit c25ae52

Browse files
dqbddqbd
authored
Merge pull request #143 from dqbd/dqbd/gpt-4.1-o3-o4
feat: add o3, o4, gpt-4.1 and missing specialised / pro models
2 parents 8963e56 + 35f522f commit c25ae52

File tree

4 files changed

+118
-8
lines changed

4 files changed

+118
-8
lines changed

‎.changeset/cuddly-comics-type.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"tiktoken": patch
3+
"js-tiktoken": patch
4+
"@dqbd/tiktoken": patch
5+
---
6+
7+
add o3, o4, gpt-4.1 and missing specialised / pro models

‎js/src/core.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -282,18 +282,43 @@ export function getEncodingNameForModel(model: TiktokenModel) {
282282
case "gpt-4o-2024-11-20":
283283
case "gpt-4o-mini-2024-07-18":
284284
case "gpt-4o-mini":
285+
case "gpt-4o-search-preview":
286+
case "gpt-4o-search-preview-2025-03-11":
287+
case "gpt-4o-mini-search-preview":
288+
case "gpt-4o-mini-search-preview-2025-03-11":
289+
case "gpt-4o-audio-preview":
290+
case "gpt-4o-audio-preview-2024-12-17":
291+
case "gpt-4o-audio-preview-2024-10-01":
292+
case "gpt-4o-mini-audio-preview":
293+
case "gpt-4o-mini-audio-preview-2024-12-17":
285294
case "o1":
286295
case "o1-2024-12-17":
287296
case "o1-mini":
297+
case "o1-mini-2024-09-12":
288298
case "o1-preview":
289299
case "o1-preview-2024-09-12":
290-
case "o1-mini-2024-09-12":
300+
case "o1-pro":
301+
case "o1-pro-2025-03-19":
302+
case "o3":
303+
case "o3-2025-04-16":
291304
case "o3-mini":
292305
case "o3-mini-2025-01-31":
306+
case "o4-mini":
307+
case "o4-mini-2025-04-16":
293308
case "chatgpt-4o-latest":
294309
case "gpt-4o-realtime":
295310
case "gpt-4o-realtime-preview-2024-10-01":
296-
{
311+
case "gpt-4o-realtime-preview-2024-12-17":
312+
case "gpt-4o-mini-realtime-preview":
313+
case "gpt-4o-mini-realtime-preview-2024-12-17":
314+
case "gpt-4.1":
315+
case "gpt-4.1-2025-04-14":
316+
case "gpt-4.1-mini":
317+
case "gpt-4.1-mini-2025-04-14":
318+
case "gpt-4.1-nano":
319+
case "gpt-4.1-nano-2025-04-14":
320+
case "gpt-4.5-preview":
321+
case "gpt-4.5-preview-2025-02-27": {
297322
return "o200k_base";
298323
}
299324
default:

‎tiktoken/model_to_encoding.json‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,41 @@
6161
"gpt-4o-2024-11-20": "o200k_base",
6262
"gpt-4o-mini-2024-07-18": "o200k_base",
6363
"gpt-4o-mini": "o200k_base",
64+
"gpt-4o-search-preview": "o200k_base",
65+
"gpt-4o-search-preview-2025-03-11": "o200k_base",
66+
"gpt-4o-mini-search-preview": "o200k_base",
67+
"gpt-4o-mini-search-preview-2025-03-11": "o200k_base",
68+
"gpt-4o-audio-preview": "o200k_base",
69+
"gpt-4o-audio-preview-2024-12-17": "o200k_base",
70+
"gpt-4o-audio-preview-2024-10-01": "o200k_base",
71+
"gpt-4o-mini-audio-preview": "o200k_base",
72+
"gpt-4o-mini-audio-preview-2024-12-17": "o200k_base",
6473
"o1": "o200k_base",
6574
"o1-2024-12-17": "o200k_base",
6675
"o1-mini": "o200k_base",
76+
"o1-mini-2024-09-12": "o200k_base",
6777
"o1-preview": "o200k_base",
6878
"o1-preview-2024-09-12": "o200k_base",
69-
"o1-mini-2024-09-12": "o200k_base",
79+
"o1-pro": "o200k_base",
80+
"o1-pro-2025-03-19": "o200k_base",
81+
"o3": "o200k_base",
82+
"o3-2025-04-16": "o200k_base",
7083
"o3-mini": "o200k_base",
7184
"o3-mini-2025-01-31": "o200k_base",
85+
"o4-mini": "o200k_base",
86+
"o4-mini-2025-04-16": "o200k_base",
7287
"chatgpt-4o-latest": "o200k_base",
7388
"gpt-4o-realtime": "o200k_base",
74-
"gpt-4o-realtime-preview-2024-10-01": "o200k_base"
89+
"gpt-4o-realtime-preview-2024-10-01": "o200k_base",
90+
"gpt-4o-realtime-preview-2024-12-17": "o200k_base",
91+
"gpt-4o-mini-realtime-preview": "o200k_base",
92+
"gpt-4o-mini-realtime-preview-2024-12-17": "o200k_base",
93+
"gpt-4.1": "o200k_base",
94+
"gpt-4.1-2025-04-14": "o200k_base",
95+
"gpt-4.1-mini": "o200k_base",
96+
"gpt-4.1-mini-2025-04-14": "o200k_base",
97+
"gpt-4.1-nano": "o200k_base",
98+
"gpt-4.1-nano-2025-04-14": "o200k_base",
99+
"gpt-4.5-preview": "o200k_base",
100+
"gpt-4.5-preview-2025-02-27": "o200k_base"
75101
}

‎wasm/src/lib.rs‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -426,17 +426,43 @@ export type TiktokenModel =
426426
| "gpt-4o-2024-11-20"
427427
| "gpt-4o-mini-2024-07-18"
428428
| "gpt-4o-mini"
429+
| "gpt-4o-search-preview"
430+
| "gpt-4o-search-preview-2025-03-11"
431+
| "gpt-4o-mini-search-preview"
432+
| "gpt-4o-mini-search-preview-2025-03-11"
433+
| "gpt-4o-audio-preview"
434+
| "gpt-4o-audio-preview-2024-12-17"
435+
| "gpt-4o-audio-preview-2024-10-01"
436+
| "gpt-4o-mini-audio-preview"
437+
| "gpt-4o-mini-audio-preview-2024-12-17"
429438
| "o1"
430439
| "o1-2024-12-17"
431440
| "o1-mini"
441+
| "o1-mini-2024-09-12"
432442
| "o1-preview"
433443
| "o1-preview-2024-09-12"
434-
| "o1-mini-2024-09-12"
444+
| "o1-pro"
445+
| "o1-pro-2025-03-19"
446+
| "o3"
447+
| "o3-2025-04-16"
435448
| "o3-mini"
436449
| "o3-mini-2025-01-31"
450+
| "o4-mini"
451+
| "o4-mini-2025-04-16"
437452
| "chatgpt-4o-latest"
438453
| "gpt-4o-realtime"
439454
| "gpt-4o-realtime-preview-2024-10-01"
455+
| "gpt-4o-realtime-preview-2024-12-17"
456+
| "gpt-4o-mini-realtime-preview"
457+
| "gpt-4o-mini-realtime-preview-2024-12-17"
458+
| "gpt-4.1"
459+
| "gpt-4.1-2025-04-14"
460+
| "gpt-4.1-mini"
461+
| "gpt-4.1-mini-2025-04-14"
462+
| "gpt-4.1-nano"
463+
| "gpt-4.1-nano-2025-04-14"
464+
| "gpt-4.5-preview"
465+
| "gpt-4.5-preview-2025-02-27";
440466
441467
/**
442468
* @param {TiktokenModel} encoding
@@ -515,17 +541,43 @@ pub fn encoding_for_model(
515541
"gpt-4o-2024-11-20" => Ok("o200k_base"),
516542
"gpt-4o-mini-2024-07-18" => Ok("o200k_base"),
517543
"gpt-4o-mini" => Ok("o200k_base"),
544+
"gpt-4o-search-preview" => Ok("o200k_base"),
545+
"gpt-4o-search-preview-2025-03-11" => Ok("o200k_base"),
546+
"gpt-4o-mini-search-preview" => Ok("o200k_base"),
547+
"gpt-4o-mini-search-preview-2025-03-11" => Ok("o200k_base"),
548+
"gpt-4o-audio-preview" => Ok("o200k_base"),
549+
"gpt-4o-audio-preview-2024-12-17" => Ok("o200k_base"),
550+
"gpt-4o-audio-preview-2024-10-01" => Ok("o200k_base"),
551+
"gpt-4o-mini-audio-preview" => Ok("o200k_base"),
552+
"gpt-4o-mini-audio-preview-2024-12-17" => Ok("o200k_base"),
518553
"o1" => Ok("o200k_base"),
519554
"o1-2024-12-17" => Ok("o200k_base"),
520555
"o1-mini" => Ok("o200k_base"),
556+
"o1-mini-2024-09-12" => Ok("o200k_base"),
521557
"o1-preview" => Ok("o200k_base"),
522558
"o1-preview-2024-09-12" => Ok("o200k_base"),
523-
"o1-mini-2024-09-12" => Ok("o200k_base"),
559+
"o1-pro" => Ok("o200k_base"),
560+
"o1-pro-2025-03-19" => Ok("o200k_base"),
561+
"o3" => Ok("o200k_base"),
562+
"o3-2025-04-16" => Ok("o200k_base"),
563+
"o3-mini" => Ok("o200k_base"),
564+
"o3-mini-2025-01-31" => Ok("o200k_base"),
565+
"o4-mini" => Ok("o200k_base"),
566+
"o4-mini-2025-04-16" => Ok("o200k_base"),
524567
"chatgpt-4o-latest" => Ok("o200k_base"),
525568
"gpt-4o-realtime" => Ok("o200k_base"),
526569
"gpt-4o-realtime-preview-2024-10-01" => Ok("o200k_base"),
527-
"o3-mini" => Ok("o200k_base"),
528-
"o3-mini-2025-01-31" => Ok("o200k_base"),
570+
"gpt-4o-realtime-preview-2024-12-17" => Ok("o200k_base"),
571+
"gpt-4o-mini-realtime-preview" => Ok("o200k_base"),
572+
"gpt-4o-mini-realtime-preview-2024-12-17" => Ok("o200k_base"),
573+
"gpt-4.1" => Ok("o200k_base"),
574+
"gpt-4.1-2025-04-14" => Ok("o200k_base"),
575+
"gpt-4.1-mini" => Ok("o200k_base"),
576+
"gpt-4.1-mini-2025-04-14" => Ok("o200k_base"),
577+
"gpt-4.1-nano" => Ok("o200k_base"),
578+
"gpt-4.1-nano-2025-04-14" => Ok("o200k_base"),
579+
"gpt-4.5-preview" => Ok("o200k_base"),
580+
"gpt-4.5-preview-2025-02-27" => Ok("o200k_base"),
529581
model => Err(JsError::new(
530582
format!("Invalid model: {}", model.to_string()).as_str(),
531583
)),

0 commit comments

Comments
 (0)