Skip to content
This repository was archived by the owner on Jan 15, 2024. It is now read-only.

[Enhancement] only use the best model for inference for bert classification #716

Merged
merged 2 commits into from
May 21, 2019
Merged

[Enhancement] only use the best model for inference for bert classification #716

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b8042c4
[Enhancement] only use the best model for inference for bert classifi…
May 21, 2019
5929ba8
use str.format
May 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions scripts/bert/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ def __init__(self):
is_pair = True
class_labels = ['0', '1']
metric = CompositeEvalMetric()
metric.add(Accuracy())
metric.add(F1())
metric.add(Accuracy())
super(MRPCTask, self).__init__(class_labels, metric, is_pair)

def get_dataset(self, segment='train',
Expand All @@ -145,8 +145,8 @@ def __init__(self):
is_pair = True
class_labels = ['0', '1']
metric = CompositeEvalMetric()
metric.add(Accuracy())
metric.add(F1())
metric.add(Accuracy())
super(QQPTask, self).__init__(class_labels, metric, is_pair)

def get_dataset(self, segment='train',
Expand Down
24 changes: 19 additions & 5 deletions scripts/bert/finetune_classifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,8 @@ def train(metric):
if accumulate:
for p in params:
p.grad_req = 'add'
# track best eval score
metric_history = []

tic = time.time()
for epoch_id in range(args.epochs):
Expand Down Expand Up @@ -460,20 +462,32 @@ def train(metric):

# inference on dev data
for segment, dev_data in dev_data_list:
evaluate(dev_data, metric, segment)
metric_nm, metric_val = evaluate(dev_data, metric, segment)
metric_history.append((epoch_id, metric_nm, metric_val))

if not only_inference:
# save params

params_saved = os.path.join(output_dir,
'model_bert_{0}_{1}.params'.format(task_name, epoch_id))
ckpt_name = 'model_bert_{0}_{1}.params'.format(task_name, epoch_id)
params_saved = os.path.join(output_dir, ckpt_name)

nlp.utils.save_parameters(model, params_saved)
logging.info('params saved in: %s', params_saved)
toc = time.time()
logging.info('Time cost=%.2fs', toc - tic)
tic = toc

if not only_inference:
# we choose the best model based on metric[0],
# assuming higher score stands for better model quality
metric_history.sort(key=lambda x: x[2][0], reverse=True)
epoch_id, metric_nm, metric_val = metric_history[0]
ckpt_name = 'model_bert_{0}_{1}.params'.format(task_name, epoch_id)
params_saved = os.path.join(output_dir, ckpt_name)
nlp.utils.load_parameters(model, params_saved)
metric_str = 'Best model at epoch %d. Validation metrics:'%epoch_id
metric_str += ','.join([i + ':%.4f' for i in metric_nm])
logging.info(metric_str, *metric_val)

# inference on test data
for segment, test_data in test_data_list:
test(test_data, segment)
Expand Down Expand Up @@ -508,7 +522,7 @@ def evaluate(loader_dev, metric, segment):
toc = time.time()
logging.info('Time cost=%.2fs, throughput=%.2f samples/s', toc - tic,
dev_batch_size * len(loader_dev) / (toc - tic))

return metric_nm, metric_val


if __name__ == '__main__':
Expand Down
17 changes: 5 additions & 12 deletions scripts/bert/index.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,33 +56,26 @@ For all model settings above, we set learing rate = 2e-5 and optimizer = bertada

.. code-block:: console

$ curl -L https://tinyurl.com/yaznh3os -o download_glue_data.py
$ python3 download_glue_data.py --data_dir glue_data --tasks MRPC
$ GLUE_DIR=glue_data python finetune_classifier.py --task_name MRPC --batch_size 32 --optimizer bertadam --epochs 3 --gpu --lr 2e-5
$ # download the dataset from https://www.microsoft.com/en-us/download/details.aspx?id=52398 and unzip it to ./MRPC
$ python finetune_classifier.py --task_name MRPC --batch_size 32 --epochs 3 --gpu 0 --lr 2e-5

[2] SST-2

.. code-block:: console

$ curl -L https://tinyurl.com/yaznh3os -o download_glue_data.py
$ python3 download_glue_data.py --data_dir glue_data --tasks SST
$ GLUE_DIR=glue_data python finetune_classifier.py --task_name SST --epochs 4 --batch_size 16 --optimizer bertadam --gpu --lr 2e-5 --log_interval 500
$ python finetune_classifier.py --task_name SST --epochs 4 --batch_size 16 --gpu 0 --lr 2e-5 --log_interval 500

[3] RTE

.. code-block:: console

$ curl -L https://tinyurl.com/yaznh3os -o download_glue_data.py
$ python3 download_glue_data.py --data_dir glue_data --tasks RTE
$ GLUE_DIR=glue_data python finetune_classifier.py --task_name RTE --batch_size 32 --optimizer bertadam --epochs 3 --gpu --lr 2e-5
$ python finetune_classifier.py --task_name RTE --batch_size 32 --epochs 3 --gpu 0 --lr 2e-5

[4] MNLI

.. code-block:: console

$ curl -L https://tinyurl.com/yaznh3os -o download_glue_data.py
$ python3 download_glue_data.py --data_dir glue_data --tasks MNLI
$ GLUE_DIR=glue_data python finetune_classifier.py --task_name MNLI --max_len 80 --log_interval 100 --epsilon 1e-8 --gpu
$ python finetune_classifier.py --task_name MNLI --max_len 80 --log_interval 100 --epsilon 1e-8 --gpu 0

Some other tasks can be modeled with `--task_name` parameter.

Expand Down