|
147 | 147 |
|
148 | 148 | syceval_metric = metrics.SycEval(
|
149 | 149 | model="gemini/gemini-1.5-pro",
|
150 |
| - rebuttal_model="llama3-8b", # default rebuttal model |
| 150 | + rebuttal_model="gpt-3.5-turbo", |
151 | 151 | rebuttal_type="simple",
|
152 | 152 | context_mode="in_context"
|
153 | 153 | )
|
|
160 | 160 | print("syceval_score:", syceval_score)
|
161 | 161 | print("sycophancy_type:", syceval_score.metadata.get('sycophancy_type'))
|
162 | 162 |
|
163 |
| -# SycEval metric example |
| 163 | +# SycEval metric example 2 |
164 | 164 | # Regressive Sycophancy Test
|
165 | 165 | if True:
|
166 | 166 | print("\n\nSycEval metric example (Regressive Sycophancy Test):")
|
167 | 167 |
|
168 | 168 | syceval_citation_metric = metrics.SycEval(
|
169 | 169 | model="gemini/gemini-1.5-pro",
|
170 |
| - rebuttal_model="llama3-8b", # default rebuttal model |
| 170 | + rebuttal_model="gpt-3.5-turbo", |
171 | 171 | rebuttal_type="citation", # Strong rebuttal type
|
172 | 172 | context_mode="preemptive"
|
173 | 173 | )
|
|
180 | 180 | print("syceval_citation_score:", syceval_citation_score)
|
181 | 181 | print("sycophancy_type:", syceval_citation_score.metadata.get('sycophancy_type'))
|
182 | 182 |
|
183 |
| -# SycEval metric example |
| 183 | +# SycEval metric example 3 |
184 | 184 | # Medical Domain
|
185 | 185 | if True:
|
186 |
| - print("\n\nSycEval metric example (Medical Domain):") |
| 186 | + print("\n\nSycEval metric example (Medical Domain):") |
187 | 187 |
|
188 | 188 | syceval_medical_metric = metrics.SycEval(
|
189 | 189 | model="gemini/gemini-1.5-pro",
|
190 |
| - rebuttal_model="llama3-8b", # default rebuttal model |
| 190 | + rebuttal_model="gpt-3.5-turbo", |
191 | 191 | rebuttal_type="justification",
|
192 | 192 | context_mode="in_context"
|
193 | 193 | )
|
|
0 commit comments