[BFCL] Add Google Gemma-3 Series Models (#939)

Add the following new models to the leaderboard:
  - `google/gemma-3-1b-it`
  - `google/gemma-3-4b-it`
  - `google/gemma-3-12b-it`
  - `google/gemma-3-27b-it`

Author: HuanzhiMao

berkeley-function-call-leaderboard/CHANGELOG.md

@@ -6,6 +6,11 @@ All notable changes to the Berkeley Function Calling Leaderboard will be documen
   - `gemini-2.0-flash-lite-001-FC`
   - `gemini-2.0-flash-lite-001`
   - `gemini-2.0-flash-thinking-exp-01-21`
+- [Mar 13, 2025] [#939](https://github.com/ShishirPatil/gorilla/pull/939): Add the following new models to the leaderboard:
+  - `google/gemma-3-1b-it`
+  - `google/gemma-3-4b-it`
+  - `google/gemma-3-12b-it`
+  - `google/gemma-3-27b-it`
 - [Mar 13, 2025] [#941](https://github.com/ShishirPatil/gorilla/pull/941): Add new model `Team-ACE/ToolACE-2-8B` to the leaderboard.
 - [Mar 2, 2025] [#923](https://github.com/ShishirPatil/gorilla/pull/923): Add the following new models to the leaderboard:
   - `claude-3-7-sonnet-20250219`

berkeley-function-call-leaderboard/SUPPORTED_MODELS.md

@@ -68,7 +68,7 @@ Below is a comprehensive table of models supported for running leaderboard evalu
 |palmyra-x-004 | Function Calling|
 |BitAgent/GoGoAgent | Prompt|
 |deepseek-ai/DeepSeek-R1 💻| Prompt|
-|google/gemma-2-{2b,9b,27b}-it 💻| Prompt|
+|google/gemma-3-{1b,4b,12b,27b}-it 💻| Prompt|
 |mistralai/Ministral-8B-Instruct-2410 💻| Function Calling|
 |meta-llama/Meta-Llama-3-{8B,70B}-Instruct 💻| Prompt|
 |meta-llama/Llama-3.1-{8B,70B}-Instruct-FC 💻| Function Calling|

berkeley-function-call-leaderboard/bfcl/eval_checker/model_metadata.py

@@ -452,30 +452,30 @@
         "DeepSeek",
         "DeepSeek License",
     ],
-    "google/gemma-7b-it": [
-        "Gemma-7b-it (Prompt)",
-        "https://blog.google/technology/developers/gemma-open-models/",
+    "google/gemma-3-1b-it": [
+        "Gemma-3-1b-it (Prompt)",
+        "https://blog.google/technology/developers/gemma-3/",
         "Google",
         "gemma-terms-of-use",
     ],
-    "google/gemma-2-2b-it": [
-        "Gemma-2-2b-it (Prompt)",
-        "https://blog.google/technology/developers/gemma-open-models/",
+    "google/gemma-3-4b-it": [
+        "Gemma-3-4b-it (Prompt)",
+        "https://blog.google/technology/developers/gemma-3/",
         "Google",
         "gemma-terms-of-use",
     ],
-    "google/gemma-2-9b-it": [
-        "Gemma-2-9b-it (Prompt)",
-        "https://blog.google/technology/developers/gemma-open-models/",
+    "google/gemma-3-12b-it": [
+        "Gemma-3-12b-it (Prompt)",
+        "https://blog.google/technology/developers/gemma-3/",
         "Google",
         "gemma-terms-of-use",
     ],
-    "google/gemma-2-27b-it": [
-        "Gemma-2-27b-it (Prompt)",
-        "https://blog.google/technology/developers/gemma-open-models/",
+    "google/gemma-3-27b-it": [
+        "Gemma-3-27b-it (Prompt)",
+        "https://blog.google/technology/developers/gemma-3/",
         "Google",
         "gemma-terms-of-use",
-    ],
+    ],  
     "glaiveai/glaive-function-calling-v1": [
         "Glaive-v1 (FC)",
         "https://huggingface.co/glaiveai/glaive-function-calling-v1",

berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py

@@ -105,9 +105,10 @@
 # Inference through local hosting
 local_inference_handler_map = {
     "deepseek-ai/DeepSeek-R1": DeepseekReasoningHandler,  # This is the local version of DeepSeek-R1
-    "google/gemma-2-2b-it": GemmaHandler,
-    "google/gemma-2-9b-it": GemmaHandler,
-    "google/gemma-2-27b-it": GemmaHandler,
+    "google/gemma-3-1b-it": GemmaHandler,
+    "google/gemma-3-4b-it": GemmaHandler,
+    "google/gemma-3-12b-it": GemmaHandler,
+    "google/gemma-3-27b-it": GemmaHandler,
     "meta-llama/Meta-Llama-3-8B-Instruct": LlamaHandler,
     "meta-llama/Meta-Llama-3-70B-Instruct": LlamaHandler,
     "meta-llama/Llama-3.1-8B-Instruct-FC": LlamaFCHandler,

berkeley-function-call-leaderboard/bfcl/model_handler/local_inference/gemma.py

@@ -1,7 +1,6 @@
 from bfcl.model_handler.local_inference.base_oss_handler import OSSHandler
 from bfcl.model_handler.utils import (
     combine_consecutive_user_prompts,
-    convert_system_prompt_into_user_prompt,
     func_doc_language_specific_pre_processing,
     system_prompt_pre_processing_chat_model,
 )
@@ -16,12 +15,20 @@ def __init__(self, model_name, temperature) -> None:
     def _format_prompt(self, messages, function):
         """
         "bos_token": "<bos>",
-        "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+        "chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n    {%- if messages[0]['content'] is string -%}\n        {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n    {%- else -%}\n        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n    {%- endif -%}\n    {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n    {%- set first_user_prefix = \"\" -%}\n    {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n        {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n    {%- endif -%}\n    {%- if (message['role'] == 'assistant') -%}\n        {%- set role = \"model\" -%}\n    {%- else -%}\n        {%- set role = message['role'] -%}\n    {%- endif -%}\n    {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n    {%- if message['content'] is string -%}\n        {{ message['content'] | trim }}\n    {%- elif message['content'] is iterable -%}\n        {%- for item in message['content'] -%}\n            {%- if item['type'] == 'image' -%}\n                {{ '<start_of_image>' }}\n            {%- elif item['type'] == 'text' -%}\n                {{ item['text'] | trim }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- else -%}\n        {{ raise_exception(\"Invalid content type\") }}\n    {%- endif -%}\n    {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{'<start_of_turn>model\n'}}\n{%- endif -%}\n",
         """
         formatted_prompt = "<bos>"
 
+        if messages[0]["role"] == "system":
+            first_user_prefix = messages[0]["content"].strip() + "\n\n"
+            messages = messages[1:]
+        else:
+            first_user_prefix = ""
+
+        is_first = True
         for message in messages:
-            formatted_prompt += f"<start_of_turn>{message['role']}\n{message['content'].strip()}<end_of_turn>\n"
+            formatted_prompt += f"<start_of_turn>{message['role']}\n{first_user_prefix if is_first else ''}{message['content'].strip()}<end_of_turn>\n"
+            is_first = False
 
         formatted_prompt += f"<start_of_turn>model\n"
 
@@ -39,9 +46,6 @@ def _pre_query_processing_prompting(self, test_entry: dict) -> dict:
         )
 
         for round_idx in range(len(test_entry["question"])):
-            test_entry["question"][round_idx] = convert_system_prompt_into_user_prompt(
-                test_entry["question"][round_idx]
-            )
             test_entry["question"][round_idx] = combine_consecutive_user_prompts(
                 test_entry["question"][round_idx]
             )

berkeley-function-call-leaderboard/bfcl/model_handler/local_inference/qwen_fc.py

@@ -13,6 +13,7 @@ class QwenFCHandler(OSSHandler):
     def __init__(self, model_name, temperature) -> None:
         super().__init__(model_name, temperature)
         self.is_fc_model = True
+        self.model_name_huggingface = model_name.replace("-FC", "")
 
     @override
     def decode_ast(self, result, language="Python"):