|
| 1 | +import os |
| 2 | + |
| 3 | +from datasets import Dataset, load_dataset |
| 4 | + |
| 5 | +from opencompass.datasets.supergpqa.supergpqa_eval import ( |
| 6 | + extract_option_content, extract_option_labels) |
| 7 | +from opencompass.datasets.supergpqa.supergpqa_utils import load_yaml |
| 8 | +from opencompass.openicl.icl_evaluator import BaseEvaluator |
| 9 | +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET |
| 10 | +from opencompass.utils import get_data_path |
| 11 | + |
| 12 | +from ..base import BaseDataset |
| 13 | + |
| 14 | + |
| 15 | +def _parse(item, template, prompt_mode): |
| 16 | + prompt_format = [ |
| 17 | + item['question'] + '\n' + '\n'.join([ |
| 18 | + f'{chr(65+i)}) {option}' |
| 19 | + for i, option in enumerate(item['options']) |
| 20 | + ]) |
| 21 | + ] |
| 22 | + item['infer_prompt'] = template['prompt_format'][0].format(*prompt_format) |
| 23 | + item['prompt_mode'] = prompt_mode |
| 24 | + return item |
| 25 | + |
| 26 | + |
| 27 | +@LOAD_DATASET.register_module() |
| 28 | +class SuperGPQADataset(BaseDataset): |
| 29 | + |
| 30 | + @staticmethod |
| 31 | + def load(path: str, prompt_mode: str, **kwargs): |
| 32 | + path = get_data_path(path, local_mode=True) |
| 33 | + dataset = load_dataset(path, split='train') |
| 34 | + |
| 35 | + # get prompt template |
| 36 | + template_path = None |
| 37 | + if prompt_mode == 'zero-shot': |
| 38 | + template_path = os.path.join( |
| 39 | + os.path.dirname(__file__), |
| 40 | + 'supergpqa_dataset_config/prompt/zero-shot.yaml', |
| 41 | + ) |
| 42 | + elif prompt_mode == 'five-shot': |
| 43 | + template_path = os.path.join( |
| 44 | + os.path.dirname(__file__), |
| 45 | + 'supergpqa_dataset_config/prompt/five-shot.yaml', |
| 46 | + ) |
| 47 | + try: |
| 48 | + template = load_yaml(template_path) |
| 49 | + except FileNotFoundError: |
| 50 | + print(f'[ERROR] Missing prompt template: {template_path}') |
| 51 | + return Dataset.from_list([]) |
| 52 | + |
| 53 | + dataset = dataset.map(lambda item: _parse(item, template, prompt_mode)) |
| 54 | + return dataset |
| 55 | + |
| 56 | + |
| 57 | +@ICL_EVALUATORS.register_module() |
| 58 | +class SuperGPQAEvaluator(BaseEvaluator): |
| 59 | + |
| 60 | + def __init__(self): |
| 61 | + super().__init__() |
| 62 | + |
| 63 | + def score(self, predictions, references, test_set): |
| 64 | + mode = test_set[0]['prompt_mode'] |
| 65 | + acc = 0 |
| 66 | + count = 0 |
| 67 | + err = 0 |
| 68 | + miss = 0 |
| 69 | + acc_difficulty = {'hard': 0, 'middle': 0, 'easy': 0} |
| 70 | + count_difficulty = {'hard': 0, 'middle': 0, 'easy': 0} |
| 71 | + stats = {'discipline': {}, 'field': {}, 'subfield': {}} |
| 72 | + details = [] |
| 73 | + for i, sample in enumerate(test_set): |
| 74 | + sample['pred'] = prediction = predictions[i] |
| 75 | + gold = references[i] |
| 76 | + if mode == 'zero-shot': |
| 77 | + predict = extract_option_labels(prediction, 'ABCDEFGHIJ') |
| 78 | + if predict is None: |
| 79 | + predict = extract_option_content(prediction, |
| 80 | + sample['options']) |
| 81 | + predict = (chr(sample['options'].index(predict) + |
| 82 | + 65) if predict else None) |
| 83 | + sample['extracted_answer'] = predict |
| 84 | + elif mode == 'five-shot': |
| 85 | + response = prediction.split('Question:')[0] |
| 86 | + predict = extract_option_labels(response, 'ABCDEFGHIJ') |
| 87 | + if predict is None: |
| 88 | + predict = extract_option_content(response, |
| 89 | + sample['options']) |
| 90 | + predict = (chr(sample['options'].index(predict) + |
| 91 | + 65) if predict else None) |
| 92 | + if predict is None: |
| 93 | + predict = extract_option_labels(prediction, 'ABCDEFGHIJ') |
| 94 | + if predict is None: |
| 95 | + predict = extract_option_content( |
| 96 | + prediction, sample['options']) |
| 97 | + predict = (chr(sample['options'].index(predict) + |
| 98 | + 65) if predict else None) |
| 99 | + sample['extracted_answer'] = predict |
| 100 | + |
| 101 | + discipline = sample.get('discipline', 'unknown') |
| 102 | + field = sample.get('field', 'unknown') |
| 103 | + subfield = sample.get('subfield', 'unknown') |
| 104 | + difficulty = sample.get('difficulty', 'unknown') |
| 105 | + |
| 106 | + for level, key in [ |
| 107 | + ('discipline', discipline), |
| 108 | + # ('field', f"{discipline}/{field}"), |
| 109 | + # ('subfield', f"{discipline}/{field}/{subfield}"), |
| 110 | + ]: |
| 111 | + if key not in stats[level]: |
| 112 | + stats[level][key] = { |
| 113 | + 'correct': 0, |
| 114 | + 'total': 0, |
| 115 | + 'miss': 0, |
| 116 | + 'error': 0, |
| 117 | + 'discipline': discipline, |
| 118 | + 'field': field, |
| 119 | + 'subfield': subfield, |
| 120 | + 'difficulty': { |
| 121 | + 'easy': { |
| 122 | + 'correct': 0, |
| 123 | + 'total': 0 |
| 124 | + }, |
| 125 | + 'middle': { |
| 126 | + 'correct': 0, |
| 127 | + 'total': 0 |
| 128 | + }, |
| 129 | + 'hard': { |
| 130 | + 'correct': 0, |
| 131 | + 'total': 0 |
| 132 | + }, |
| 133 | + }, |
| 134 | + } |
| 135 | + |
| 136 | + stats[level][key]['total'] += 1 |
| 137 | + stats[level][key]['difficulty'][difficulty]['total'] += 1 |
| 138 | + |
| 139 | + answer_letter = sample['answer_letter'] |
| 140 | + assert answer_letter == gold |
| 141 | + if predict and answer_letter == predict: |
| 142 | + acc += 1 |
| 143 | + acc_difficulty[difficulty] += 1 |
| 144 | + sample['status'] = 'correct' |
| 145 | + stats[level][key]['correct'] += 1 |
| 146 | + stats[level][key]['difficulty'][difficulty]['correct'] += 1 |
| 147 | + elif predict is None or predict == '': |
| 148 | + miss += 1 |
| 149 | + sample['status'] = 'miss' |
| 150 | + stats[level][key]['miss'] += 1 |
| 151 | + elif predict == 'error': |
| 152 | + err += 1 |
| 153 | + sample['status'] = 'error' |
| 154 | + stats[level][key]['error'] += 1 |
| 155 | + else: |
| 156 | + sample['status'] = 'incorrect' |
| 157 | + count += 1 |
| 158 | + count_difficulty[difficulty] += 1 |
| 159 | + details.append({ |
| 160 | + 'pred': sample['pred'], |
| 161 | + 'answer': sample['answer'], |
| 162 | + 'parsed_answer': sample['extracted_answer'], |
| 163 | + 'correct': True if sample['status'] else False, |
| 164 | + }) |
| 165 | + |
| 166 | + return { |
| 167 | + 'accuracy': |
| 168 | + acc / count if count > 0 else 0, |
| 169 | + 'error_rate': |
| 170 | + err / count if count > 0 else 0, |
| 171 | + 'miss_rate': |
| 172 | + miss / count if count > 0 else 0, |
| 173 | + 'hard_accuracy': |
| 174 | + (acc_difficulty['hard'] / |
| 175 | + count_difficulty['hard'] if count_difficulty['hard'] > 0 else 0), |
| 176 | + 'middle_accuracy': |
| 177 | + (acc_difficulty['middle'] / count_difficulty['middle'] |
| 178 | + if count_difficulty['middle'] > 0 else 0), |
| 179 | + 'easy_accuracy': |
| 180 | + (acc_difficulty['easy'] / |
| 181 | + count_difficulty['easy'] if count_difficulty['easy'] > 0 else 0), |
| 182 | + 'details': |
| 183 | + details, |
| 184 | + } |
0 commit comments