Skip to content

Commit 0a3e3f7

Browse files
arvinxxarvinxx
authored
🐛 fix: fix Google Gemini pro 1.5 and system role not take effect (#1801)
* 🐛 fix: fix google gemini pro 1.5 * Update index.ts * 1.5 support vision * Update index.ts * Update index.ts * 🐛 fix: improve google model * 🐛 fix: 修正 google 对话没有配对的问题 * 🐛 fix: 修正 vision 判断 * 🐛 fix: improve model card and region * 🐛 fix: fix vision * hide ultra model
1 parent 11013a7 commit 0a3e3f7

File tree

6 files changed

+246
-81
lines changed

6 files changed

+246
-81
lines changed

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@
8484
"@aws-sdk/client-bedrock-runtime": "^3.525.0",
8585
"@azure/openai": "^1.0.0-beta.11",
8686
"@cfworker/json-schema": "^1",
87-
"@google/generative-ai": "^0.2.0",
87+
"@google/generative-ai": "^0.3.1",
8888
"@icons-pack/react-simple-icons": "^9",
8989
"@lobehub/chat-plugin-sdk": "latest",
9090
"@lobehub/chat-plugins-gateway": "latest",

‎src/app/api/chat/google/route.ts

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,7 @@ import { POST as UniverseRoute } from '../[provider]/route';
1313
// so if you want to use with proxy, you need comment the code below
1414
export const runtime = 'edge';
1515

16-
export const preferredRegion = [
17-
'bom1',
18-
'cle1',
19-
'cpt1',
20-
'gru1',
21-
'hnd1',
22-
'iad1',
23-
'icn1',
24-
'kix1',
25-
'pdx1',
26-
'sfo1',
27-
'sin1',
28-
'syd1',
29-
];
16+
// due to gemini-1.5-pro only can be used in us, so we need to set the preferred region only in US
17+
export const preferredRegion = ['cle1', 'iad1', 'pdx1', 'sfo1'];
3018

3119
export const POST = async (req: Request) => UniverseRoute(req, { params: { provider: 'google' } });

‎src/config/modelProviders/google.ts

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,86 @@ import { ModelProviderCard } from '@/types/llm';
33
const Google: ModelProviderCard = {
44
chatModels: [
55
{
6-
displayName: 'Gemini Pro',
6+
description: 'A legacy text-only model optimized for chat conversations',
7+
displayName: 'PaLM 2 Chat (Legacy)',
8+
hidden: true,
9+
id: 'chat-bison-001',
10+
maxOutput: 1024,
11+
tokens: 5120,
12+
},
13+
{
14+
description: 'A legacy model that understands text and generates text as an output',
15+
displayName: 'PaLM 2 (Legacy)',
16+
hidden: true,
17+
id: 'text-bison-001',
18+
maxOutput: 1024,
19+
tokens: 9220,
20+
},
21+
{
22+
description: 'The best model for scaling across a wide range of tasks',
23+
displayName: 'Gemini 1.0 Pro',
724
id: 'gemini-pro',
8-
tokens: 30_720,
25+
maxOutput: 2048,
26+
tokens: 32_768,
27+
},
28+
{
29+
description: 'The best image understanding model to handle a broad range of applications',
30+
displayName: 'Gemini 1.0 Pro Vision',
31+
id: 'gemini-1.0-pro-vision-latest',
32+
maxOutput: 4096,
33+
tokens: 16_384,
34+
vision: true,
935
},
1036
{
11-
displayName: 'Gemini Pro Vision',
37+
description: 'The best image understanding model to handle a broad range of applications',
38+
displayName: 'Gemini 1.0 Pro Vision',
39+
hidden: true,
1240
id: 'gemini-pro-vision',
13-
tokens: 12_288,
41+
maxOutput: 4096,
42+
tokens: 16_384,
1443
vision: true,
1544
},
1645
{
46+
description: 'The best model for scaling across a wide range of tasks',
47+
displayName: 'Gemini 1.0 Pro',
48+
hidden: true,
49+
id: '1.0-pro',
50+
maxOutput: 2048,
51+
tokens: 32_768,
52+
},
53+
{
54+
description:
55+
'The best model for scaling across a wide range of tasks. This is a stable model that supports tuning.',
56+
displayName: 'Gemini 1.0 Pro 001 (Tuning)',
57+
hidden: true,
58+
id: 'gemini-1.0-pro-001',
59+
maxOutput: 2048,
60+
tokens: 32_768,
61+
},
62+
{
63+
description:
64+
'The best model for scaling across a wide range of tasks. This is the latest model.',
65+
displayName: 'Gemini 1.0 Pro Latest',
66+
hidden: true,
67+
id: 'gemini-1.0-pro-latest',
68+
maxOutput: 2048,
69+
tokens: 32_768,
70+
},
71+
{
72+
description: 'Mid-size multimodal model that supports up to 1 million tokens',
1773
displayName: 'Gemini 1.5 Pro',
1874
id: 'gemini-1.5-pro-latest',
19-
tokens: 1_048_576,
75+
maxOutput: 8192,
76+
tokens: 1_056_768,
77+
vision: true,
2078
},
2179
{
22-
displayName: 'Gemini Ultra',
80+
description: 'The most capable model for highly complex tasks',
81+
displayName: 'Gemini 1.0 Ultra',
82+
hidden: true,
2383
id: 'gemini-ultra-latest',
24-
tokens: 30_720,
84+
maxOutput: 2048,
85+
tokens: 32_768,
2586
},
2687
],
2788
id: 'google',

‎src/libs/agent-runtime/google/index.test.ts

Lines changed: 74 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// @vitest-environment edge-runtime
2-
import { GenerateContentRequest, GenerateContentStreamResult, Part } from '@google/generative-ai';
32
import OpenAI from 'openai';
43
import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
54

@@ -317,17 +316,55 @@ describe('LobeGoogleAI', () => {
317316
});
318317

319318
describe('buildGoogleMessages', () => {
320-
it('should use default text model when no images are included in messages', () => {
319+
it('get default result with gemini-pro', () => {
320+
const messages: OpenAIChatMessage[] = [{ content: 'Hello', role: 'user' }];
321+
322+
const contents = instance['buildGoogleMessages'](messages, 'gemini-pro');
323+
324+
expect(contents).toHaveLength(1);
325+
expect(contents).toEqual([{ parts: [{ text: 'Hello' }], role: 'user' }]);
326+
});
327+
328+
it('messages should end with user if using gemini-pro', () => {
321329
const messages: OpenAIChatMessage[] = [
322330
{ content: 'Hello', role: 'user' },
323331
{ content: 'Hi', role: 'assistant' },
324332
];
325-
const model = 'text-davinci-003';
326333

327-
// 调用 buildGoogleMessages 方法
328-
const { contents, model: usedModel } = instance['buildGoogleMessages'](messages, model);
334+
const contents = instance['buildGoogleMessages'](messages, 'gemini-pro');
335+
336+
expect(contents).toHaveLength(3);
337+
expect(contents).toEqual([
338+
{ parts: [{ text: 'Hello' }], role: 'user' },
339+
{ parts: [{ text: 'Hi' }], role: 'model' },
340+
{ parts: [{ text: '' }], role: 'user' },
341+
]);
342+
});
343+
344+
it('should include system role if there is a system role prompt', () => {
345+
const messages: OpenAIChatMessage[] = [
346+
{ content: 'you are ChatGPT', role: 'system' },
347+
{ content: 'Who are you', role: 'user' },
348+
];
349+
350+
const contents = instance['buildGoogleMessages'](messages, 'gemini-pro');
351+
352+
expect(contents).toHaveLength(3);
353+
expect(contents).toEqual([
354+
{ parts: [{ text: 'you are ChatGPT' }], role: 'user' },
355+
{ parts: [{ text: '' }], role: 'model' },
356+
{ parts: [{ text: 'Who are you' }], role: 'user' },
357+
]);
358+
});
359+
360+
it('should not modify the length if model is gemini-1.5-pro', () => {
361+
const messages: OpenAIChatMessage[] = [
362+
{ content: 'Hello', role: 'user' },
363+
{ content: 'Hi', role: 'assistant' },
364+
];
365+
366+
const contents = instance['buildGoogleMessages'](messages, 'gemini-1.5-pro-latest');
329367

330-
expect(usedModel).toEqual('gemini-pro'); // 假设 'gemini-pro' 是默认文本模型
331368
expect(contents).toHaveLength(2);
332369
expect(contents).toEqual([
333370
{ parts: [{ text: 'Hello' }], role: 'user' },
@@ -348,9 +385,8 @@ describe('LobeGoogleAI', () => {
348385
const model = 'gemini-pro-vision';
349386

350387
// 调用 buildGoogleMessages 方法
351-
const { contents, model: usedModel } = instance['buildGoogleMessages'](messages, model);
388+
const contents = instance['buildGoogleMessages'](messages, model);
352389

353-
expect(usedModel).toEqual(model);
354390
expect(contents).toHaveLength(1);
355391
expect(contents).toEqual([
356392
{
@@ -360,5 +396,35 @@ describe('LobeGoogleAI', () => {
360396
]);
361397
});
362398
});
399+
400+
describe('convertModel', () => {
401+
it('should use default text model when no images are included in messages', () => {
402+
const messages: OpenAIChatMessage[] = [
403+
{ content: 'Hello', role: 'user' },
404+
{ content: 'Hi', role: 'assistant' },
405+
];
406+
407+
// 调用 buildGoogleMessages 方法
408+
const model = instance['convertModel']('gemini-pro-vision', messages);
409+
410+
expect(model).toEqual('gemini-pro'); // 假设 'gemini-pro' 是默认文本模型
411+
});
412+
413+
it('should use specified model when images are included in messages', () => {
414+
const messages: OpenAIChatMessage[] = [
415+
{
416+
content: [
417+
{ type: 'text', text: 'Hello' },
418+
{ type: 'image_url', image_url: { url: 'data:image/png;base64,...' } },
419+
],
420+
role: 'user',
421+
},
422+
];
423+
424+
const model = instance['convertModel']('gemini-pro-vision', messages);
425+
426+
expect(model).toEqual('gemini-pro-vision');
427+
});
428+
});
363429
});
364430
});

0 commit comments

Comments
 (0)