21
21
PipelineSingleFeedback = CoSTEERSingleFeedback
22
22
PipelineMultiFeedback = CoSTEERMultiFeedback
23
23
24
+ NO_SUB = "<No submission.csv file found.>"
25
+ NO_SCORE = "<No scores.csv file found.>"
26
+
24
27
25
28
class ModelDumpEvaluator (CoSTEEREvaluator ):
26
29
"""This evaluator assumes that it runs after the model"""
@@ -43,6 +46,7 @@ def evaluate(
43
46
code = err_msg ,
44
47
final_decision = False ,
45
48
)
49
+
46
50
data_source_path = (
47
51
f"{ DS_RD_SETTING .local_data_path } /{ self .scen .competition } "
48
52
if self .data_type == "full"
@@ -61,12 +65,12 @@ def evaluate(
61
65
submission_content_before = (
62
66
(implementation .workspace_path / "submission.csv" ).read_text ()
63
67
if (implementation .workspace_path / "submission.csv" ).exists ()
64
- else None
68
+ else NO_SUB
65
69
)
66
70
scores_content_before = (
67
71
(implementation .workspace_path / "scores.csv" ).read_text ()
68
72
if (implementation .workspace_path / "scores.csv" ).exists ()
69
- else None
73
+ else NO_SCORE
70
74
)
71
75
72
76
# Remove the files submission.csv and scores.csv
@@ -103,11 +107,16 @@ def evaluate(
103
107
final_decision = False ,
104
108
)
105
109
106
- assert submission_content_before is not None
107
- assert scores_content_before is not None
108
-
109
- submission_content_after = (implementation .workspace_path / "submission.csv" ).read_text ()
110
- scores_content_after = (implementation .workspace_path / "scores.csv" ).read_text ()
110
+ submission_content_after = (
111
+ (implementation .workspace_path / "submission.csv" ).read_text ()
112
+ if (implementation .workspace_path / "submission.csv" ).exists ()
113
+ else NO_SUB
114
+ )
115
+ scores_content_after = (
116
+ (implementation .workspace_path / "scores.csv" ).read_text ()
117
+ if (implementation .workspace_path / "scores.csv" ).exists ()
118
+ else NO_SCORE
119
+ )
111
120
112
121
system_prompt = T (".prompts:dump_model_eval.system" ).r ()
113
122
user_prompt = T (".prompts:dump_model_eval.user" ).r (
0 commit comments