Skip to content

Commit 2f39693

Browse files
pgarbackipgarbacki
authored
Add firefunction-v2 to the leaderboard (#470)
This PR adds firefunction-v2 to the leaderboard. It also updates the fireworks handler making sure that the parameters and prompt are set correctly.
1 parent 67039bd commit 2f39693

File tree

4 files changed

+69
-13
lines changed

4 files changed

+69
-13
lines changed

‎berkeley-function-call-leaderboard/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ Below is *a table of models we support* to run our leaderboard evaluation agains
196196
|command-r-plus | Prompt|
197197
|databrick-dbrx-instruct | Prompt|
198198
|deepseek-ai/deepseek-coder-6.7b-instruct 💻| Prompt|
199-
|fire-function-v1-FC | Function Calling|
199+
|firefunction-{v1,v2}-FC | Function Calling|
200200
|gemini-1.0-pro | Function Calling|
201201
|gemini-1.5-pro-preview-{0409,0514} | Function Calling|
202202
|gemini-1.5-flash-preview-0514 | Function Calling|

‎berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -236,12 +236,18 @@
236236
"Nexusflow",
237237
"Apache 2.0",
238238
],
239-
"fire-function-v1-FC": [
239+
"firefunction-v1-FC": [
240240
"FireFunction-v1 (FC)",
241241
"https://huggingface.co/fireworks-ai/firefunction-v1",
242242
"Fireworks",
243243
"Apache 2.0",
244244
],
245+
"firefunction-v2-FC": [
246+
"FireFunction-v2 (FC)",
247+
"https://huggingface.co/fireworks-ai/firefunction-v2",
248+
"Fireworks",
249+
"Apache 2.0",
250+
],
245251
"gemini-1.5-pro-preview-0514": [
246252
"Gemini-1.5-Pro-Preview-0514 (FC)",
247253
"https://deepmind.google/technologies/gemini/pro/",
@@ -349,7 +355,7 @@
349355
"https://huggingface.co/Snowflake/snowflake-arctic-instruct",
350356
"Snowflake",
351357
"apache-2.0",
352-
]
358+
],
353359
}
354360

355361
INPUT_PRICE_PER_MILLION_TOKEN = {
@@ -445,7 +451,8 @@
445451

446452
NO_COST_MODELS = [
447453
"Nexusflow-Raven-v2",
448-
"fire-function-v1-FC",
454+
"firefunction-v1-FC",
455+
"firefunction-v2-FC",
449456
"meetkai/functionary-medium-v2.4-FC",
450457
"meetkai/functionary-small-v2.2-FC",
451458
"meetkai/functionary-small-v2.4-FC",

‎berkeley-function-call-leaderboard/model_handler/firework_ai_handler.py

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
1+
import json
2+
import os
3+
import time
4+
5+
from model_handler.constant import GORILLA_TO_OPENAPI
16
from model_handler.gpt_handler import OpenAIHandler
27
from model_handler.model_style import ModelStyle
3-
import os, json
8+
from model_handler.utils import convert_to_tool, language_specific_pre_processing
49
from openai import OpenAI
510

611

712
class FireworkAIHandler(OpenAIHandler):
8-
def __init__(self, model_name, temperature=0.7, top_p=1, max_tokens=1000) -> None:
13+
def __init__(self, model_name, temperature=0.0, top_p=1, max_tokens=1000) -> None:
914
super().__init__(model_name, temperature, top_p, max_tokens)
10-
self.model_name = "accounts/fireworks/models/firefunction-v1-FC"
1115
self.model_style = ModelStyle.FIREWORK_AI
16+
self.temperature = 0.0
1217

1318
self.client = OpenAI(
1419
base_url="https://api.fireworks.ai/inference/v1",
@@ -19,11 +24,54 @@ def write(self, result, file_to_open):
1924
# This method is used to write the result to the file.
2025
if not os.path.exists("./result"):
2126
os.mkdir("./result")
22-
if not os.path.exists("./result/fire-function-v1-FC"):
23-
os.mkdir("./result/fire-function-v1-FC")
27+
if not os.path.exists(f"./result/{self.model_name}"):
28+
os.mkdir(f"./result/{self.model_name}")
2429
with open(
25-
"./result/fire-function-v1-FC/"
30+
f"./result/{self.model_name}/"
2631
+ file_to_open.replace(".json", "_result.json"),
2732
"a+",
2833
) as f:
2934
f.write(json.dumps(result) + "\n")
35+
36+
def inference(self, prompt, functions, test_category):
37+
functions = language_specific_pre_processing(functions, test_category, True)
38+
if type(functions) is not list:
39+
functions = [functions]
40+
message = [{"role": "user", "content": prompt}]
41+
oai_tool = convert_to_tool(
42+
functions, GORILLA_TO_OPENAPI, self.model_style, test_category, True
43+
)
44+
start_time = time.time()
45+
model_name = self.model_name.replace("-FC", "")
46+
model_name = f"accounts/fireworks/models/{model_name}"
47+
if len(oai_tool) > 0:
48+
response = self.client.chat.completions.create(
49+
messages=message,
50+
model=model_name,
51+
temperature=self.temperature,
52+
max_tokens=self.max_tokens,
53+
top_p=self.top_p,
54+
tools=oai_tool,
55+
frequency_penalty=0.4,
56+
)
57+
else:
58+
response = self.client.chat.completions.create(
59+
messages=message,
60+
model=model_name,
61+
temperature=self.temperature,
62+
max_tokens=self.max_tokens,
63+
top_p=self.top_p,
64+
)
65+
latency = time.time() - start_time
66+
try:
67+
result = [
68+
{func_call.function.name: func_call.function.arguments}
69+
for func_call in response.choices[0].message.tool_calls
70+
]
71+
except:
72+
result = response.choices[0].message.content
73+
metadata = {}
74+
metadata["input_tokens"] = response.usage.prompt_tokens
75+
metadata["output_tokens"] = response.usage.completion_tokens
76+
metadata["latency"] = latency
77+
return result, metadata

‎berkeley-function-call-leaderboard/model_handler/handler_map.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1+
from model_handler.arctic_handler import ArcticHandler
12
from model_handler.claude_fc_handler import ClaudeFCHandler
23
from model_handler.claude_prompt_handler import ClaudePromptingHandler
4+
from model_handler.cohere_handler import CohereHandler
35
from model_handler.databricks_handler import DatabricksHandler
46
from model_handler.deepseek_handler import DeepseekHandler
57
from model_handler.firework_ai_handler import FireworkAIHandler
@@ -14,8 +16,6 @@
1416
from model_handler.mistral_handler import MistralHandler
1517
from model_handler.nexus_handler import NexusHandler
1618
from model_handler.oss_handler import OSSHandler
17-
from model_handler.cohere_handler import CohereHandler
18-
from model_handler.arctic_handler import ArcticHandler
1919

2020
handler_map = {
2121
"gorilla-openfunctions-v0": GorillaHandler,
@@ -48,7 +48,8 @@
4848
"mistral-small-2402-FC-Any": MistralHandler,
4949
"mistral-small-2402-FC-Auto": MistralHandler,
5050
"mistral-tiny-2312": MistralHandler,
51-
"fire-function-v1-FC": FireworkAIHandler,
51+
"firefunction-v1-FC": FireworkAIHandler,
52+
"firefunction-v2-FC": FireworkAIHandler,
5253
"Nexusflow-Raven-v2": NexusHandler,
5354
"gemini-1.0-pro": GeminiHandler,
5455
"gemini-1.5-pro-preview-0409": GeminiHandler,

0 commit comments

Comments
 (0)