@@ -295,9 +295,12 @@ def preprocess_data(tokenizer, task, batch_size, dev_batch_size, max_len, pad=Fa
295
295
data_train_len = data_train .transform (
296
296
lambda input_id , length , segment_id , label_id : length , lazy = False )
297
297
# bucket sampler for training
298
+ pad_val = vocabulary [vocabulary .padding_token ]
298
299
batchify_fn = nlp .data .batchify .Tuple (
299
- nlp .data .batchify .Pad (axis = 0 ), nlp .data .batchify .Stack (),
300
- nlp .data .batchify .Pad (axis = 0 ), nlp .data .batchify .Stack (label_dtype ))
300
+ nlp .data .batchify .Pad (axis = 0 , pad_val = pad_val ), # input
301
+ nlp .data .batchify .Stack (), # length
302
+ nlp .data .batchify .Pad (axis = 0 , pad_val = 0 ), # segment
303
+ nlp .data .batchify .Stack (label_dtype )) # label
301
304
batch_sampler = nlp .data .sampler .FixedBucketSampler (
302
305
data_train_len ,
303
306
batch_size = batch_size ,
@@ -327,8 +330,8 @@ def preprocess_data(tokenizer, task, batch_size, dev_batch_size, max_len, pad=Fa
327
330
328
331
# batchify for data test
329
332
test_batchify_fn = nlp .data .batchify .Tuple (
330
- nlp .data .batchify .Pad (axis = 0 ), nlp .data .batchify .Stack (),
331
- nlp .data .batchify .Pad (axis = 0 ))
333
+ nlp .data .batchify .Pad (axis = 0 , pad_val = pad_val ), nlp .data .batchify .Stack (),
334
+ nlp .data .batchify .Pad (axis = 0 , pad_val = 0 ))
332
335
# transform for data test
333
336
test_trans = BERTDatasetTransform (tokenizer , max_len ,
334
337
class_labels = None ,
0 commit comments