Merge branch 'develop' into t5

Author: FrostML

applications/document_intelligence/doc_vqa/Extraction/run_docvqa.py

@@ -12,25 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import sys
-import copy
+import argparse
 import json
-import random
 import logging
+import os
+import random
 import warnings
-import argparse
-import numpy as np
-from collections import OrderedDict, Counter
+from collections import Counter
 
+import numpy as np
 import paddle
-from paddle.static import InputSpec
-from paddle.jit import to_static
-from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer
-
 from docvqa import DocVQA
 from model import LayoutXLMForTokenClassification_with_CRF
 
+from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer
+
 warnings.filterwarnings("ignore")
 logger = logging.getLogger(__name__)
 
@@ -166,7 +162,6 @@ def main(args):
     global_step = 0
     tr_loss = 0.0
     set_seed(args)
-    best_metrics = None
     for epoch_id in range(args.num_train_epochs):
         print("epoch id:{}".format(epoch_id))
         for step, batch in enumerate(train_dataloader):
@@ -193,7 +188,7 @@ def main(args):
                         step,
                         len(train_dataloader),
                         lr_scheduler.get_lr(),
-                        loss.numpy()[0],
+                        float(loss),
                     )
                 )
 
@@ -322,7 +317,6 @@ def _normalize(in_str):
 
 
 def calc_f1_score(answer, prediction):
-    f1_scores = []
     ans_segs = _tokenize_chinese_chars(_normalize(answer))
     prediction_segs = _tokenize_chinese_chars(_normalize(prediction))
     f1 = fast_f1(prediction_segs, ans_segs)
@@ -436,7 +430,6 @@ def evaluate(args, model, tokenizer, label2id_map, id2label_map, pad_token_label
             line_json["question"] = line_label["question"]
             line_json["label_answer"] = line_text[1]
             line_json["predict_answer"] = line_text[2]
-            all_boxes = line_res[3]
             label_bbox_index, predict_bbox_index = line_text[3], line_text[4]
             label_bboxes, predict_bboxes = [], []
             for i in range(len(line_label["bboxes"])):

applications/sentiment_analysis/ASO_analysis/demo.py

@@ -86,7 +86,7 @@ def predict(args, ext_model, cls_model, tokenizer, ext_id2label, cls_id2label):
             token_type_ids = paddle.to_tensor([encoded_inputs["token_type_ids"]])
 
             logits = cls_model(input_ids, token_type_ids=token_type_ids)
-            prediction = logits.argmax(axis=1).numpy()[0]
+            prediction = int(logits.argmax(axis=1))
 
             result = {"aspect": aspect, "opinions": opinion_words, "sentiment_polarity": cls_id2label[prediction]}
             results.append(result)

examples/dialogue/lic2021_baseline/finetune.py

@@ -12,21 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import math
 import os
 import time
-import math
+
 import paddle
 import paddle.distributed as dist
 import paddle.nn as nn
 import paddle.nn.functional as F
+from args import parse_args, print_args
+from data import DialogueDataset
 from paddle.io import DataLoader
-from paddle.optimizer.lr import NoamDecay
 from paddle.optimizer import AdamW
+from paddle.optimizer.lr import NoamDecay
 
-from paddlenlp.transformers import UnifiedTransformerLMHeadModel, UnifiedTransformerTokenizer
-
-from args import parse_args, print_args
-from data import DialogueDataset
+from paddlenlp.transformers import (
+    UnifiedTransformerLMHeadModel,
+    UnifiedTransformerTokenizer,
+)
 
 
 def save_ckpt(model, tokenizer, save_dir, name):
@@ -129,7 +132,7 @@ def evaluation(model, data_loader):
         logits = model(token_ids, type_ids, pos_ids, generation_mask, tgt_pos)
         loss = F.cross_entropy(logits, tgt_label, reduction="sum")
 
-        total_loss += loss.numpy()[0]
+        total_loss += float(loss.numpy())
         total_tokens += tgt_label.shape[0]
 
     avg_loss = total_loss / total_tokens

examples/language_model/elmo/run_eval.py

@@ -12,17 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import time
 import math
-import paddle
-import paddle.nn as nn
-from paddle.io import DataLoader
-import paddle.distributed as dist
+import time
 
+import paddle
 from args import parse_args, print_args
+from dataset import OneBillionWordDataset, load_vocab
 from elmo import ELMo, ELMoLoss
-from dataset import load_vocab, OneBillionWordDataset
+from paddle.io import DataLoader
 
 
 @paddle.no_grad()
@@ -67,14 +64,14 @@ def eval(args):
         loss = elmo_loss(outputs, [next_ids, next_ids_reverse])
         ppl = paddle.exp(loss)
 
-        total_loss += loss.numpy()[0]
+        total_loss += float(loss)
         total_step += 1
 
         total_time += time.time() - batch_start_time
         if step % args.log_freq == 0:
             print(
                 "Eval step %d - loss: %.4f - Perplexity: %.4f - %.3fs/step"
-                % (step, loss.numpy()[0] * args.unroll_steps, ppl.numpy()[0], total_time / args.log_freq)
+                % (step, float(loss) * args.unroll_steps, float(ppl), total_time / args.log_freq)
             )
             total_time = 0.0
         batch_start_time = time.time()

examples/language_model/elmo/run_finetune.py

@@ -12,21 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import paddle
-import paddle.nn as nn
-import paddle.nn.functional as F
-from paddle.io import Dataset, DataLoader
-import paddle.distributed as dist
-
+import argparse
 import os
 import re
-import argparse
-import numpy as np
-from typing import List
-from sklearn.model_selection import train_test_split
 
+import numpy as np
+import paddle
+import paddle.distributed as dist
+import paddle.nn as nn
 from dataset import load_vocab
 from elmo import get_elmo_layer
+from paddle.io import DataLoader, Dataset
+from sklearn.model_selection import train_test_split
 
 
 # yapf: disable
@@ -249,7 +246,7 @@ def finetune(args):
             adam.clear_grad()
 
             if step % args.logging_step == 0:
-                print("step {}, loss {}".format(step, loss.numpy()[0]))
+                print("step {}, loss {}".format(step, float(loss)))
 
     acc = test(model, test_loader)
     print("\ntest acc {}\n".format(acc))
@@ -268,7 +265,7 @@ def test(model, test_loader):
         num += label.shape[0]
         predict = paddle.argmax(output, axis=1)
         label = paddle.cast(label, dtype=predict.dtype)
-        correct += paddle.sum(paddle.cast(predict == label, dtype="int64")).numpy()[0]
+        correct += int(paddle.sum(paddle.cast(predict == label, dtype="int64")))
     model.train()
     return correct * 1.0 / num
 

examples/language_model/elmo/run_pretrain.py

@@ -14,14 +14,14 @@
 
 import os
 import time
+
 import paddle
-import paddle.nn as nn
-from paddle.io import DataLoader
 import paddle.distributed as dist
-
+import paddle.nn as nn
 from args import parse_args, print_args
+from dataset import OneBillionWordDataset, load_vocab
 from elmo import ELMo, ELMoLoss
-from dataset import load_vocab, OneBillionWordDataset
+from paddle.io import DataLoader
 
 
 def save_params(elmo, optimizer, save_dir, name):
@@ -104,7 +104,7 @@ def train(args):
         if step % args.log_freq == 0:
             print(
                 "step %d/%d - loss: %.4f - Perplexity: %.4f - %.3fs/step"
-                % (step, n_steps_total, loss.numpy()[0], ppl.numpy()[0], total_time / args.log_freq)
+                % (step, n_steps_total, float(loss), float(ppl), total_time / args.log_freq)
             )
             total_time = 0.0
         if rank == 0 and step % args.save_freq == 0:

examples/language_model/elmo/word2vec_base.py

@@ -12,18 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import paddle
-import paddle.nn as nn
-import paddle.nn.functional as F
-from paddle.io import Dataset, DataLoader
-import paddle.distributed as dist
-
+import argparse
 import os
 import re
-import argparse
+
 import numpy as np
-from sklearn.model_selection import train_test_split
+import paddle
+import paddle.distributed as dist
+import paddle.nn as nn
 from gensim.models.keyedvectors import KeyedVectors
+from paddle.io import DataLoader, Dataset
+from sklearn.model_selection import train_test_split
 
 
 # yapf: disable
@@ -227,7 +226,7 @@ def train(args):
             adam.clear_grad()
 
             if step % args.logging_step == 0:
-                print("step %d, loss %.4f" % (step, loss.numpy()[0]))
+                print("step %d, loss %.4f" % (step, float(loss)))
 
     acc = test(model, test_loader)
     print("\ntest acc %.4f\n" % acc)
@@ -246,7 +245,7 @@ def test(model, test_loader):
         num += label.shape[0]
         predict = paddle.argmax(output, axis=1)
         label = paddle.cast(label, dtype=predict.dtype)
-        correct += paddle.sum(paddle.cast(predict == label, dtype="int64")).numpy()[0]
+        correct += int(paddle.sum(paddle.cast(predict == label, dtype="int64")))
     model.train()
     return correct * 1.0 / num
 

examples/language_model/moe/dygraph/run_moe_pretrain.py

@@ -12,37 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import argparse
-import math
 import os
 import random
 import time
+import types
+from types import MethodType
 
+import lr
 import numpy as np
 import paddle
-from visualdl import LogWriter
-from modeling import GPTModel, GPTForPretraining, GPTPretrainingCriterion, GPTForPretrainingPipe
-from paddlenlp.transformers import GPTTokenizer, GPTChineseTokenizer
-from paddlenlp.utils.log import logger
-
-from dataset import create_pretrained_dataset
+import paddle.distributed as dist
 from args import parse_args
-import lr
+from checkpointing import load_checkpoint, save_checkpoint
+from dataset import create_pretrained_dataset
+from framework import AdamW, group_sharded_parallel, obtain_storage
+from modeling import (
+    GPTForPretraining,
+    GPTForPretrainingPipe,
+    GPTModel,
+    GPTPretrainingCriterion,
+)
+from paddle import _legacy_C_ops
 from paddle.distributed import fleet
 from paddle.distributed.fleet.meta_parallel import get_rng_state_tracker
-from paddle.distributed.fleet.utils.hybrid_parallel_util import fused_allreduce_gradients
-import types
-from utils import get_timers, set_timers
-from types import MethodType
-from paddle import _legacy_C_ops
+from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import (
+    GroupShardedScaler,
+)
+from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import (
+    ShardingScaler,
+)
 from paddle.fluid.framework import core, in_dygraph_mode
-import paddle.distributed as dist
-from framework import assign_group_by_size, flatten_dense_tensors, obtain_storage, AdamW, group_sharded_parallel
 from paddle.incubate.distributed.models import moe
-from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler
-from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler
+from utils import get_timers, set_timers
+from visualdl import LogWriter
 
-from checkpointing import save_checkpoint, load_checkpoint
+from paddlenlp.transformers import GPTChineseTokenizer, GPTTokenizer
+from paddlenlp.utils.log import logger
 
 MODEL_CLASSES = {
     "gpt": (GPTForPretraining, GPTTokenizer),
@@ -172,7 +177,7 @@ def unscale_method(self, optimizer):
     if dist.get_world_size() > 1:
         is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32")
         paddle.distributed.all_reduce(is_found_inf, op=paddle.distributed.ReduceOp.MAX, group=None)
-        self._found_inf = is_found_inf.numpy()[0]
+        self._found_inf = int(is_found_inf)
 
 
 def all_reduce_parameters(params, group):
@@ -437,7 +442,7 @@ def do_train(args):
 
     clip = None
     if args.grad_clip > 0:
-        is_expert_param_fun = lambda param: param.name in expert_fusion_names
+        is_expert_param_fun = lambda param: param.name in expert_fusion_names  # noqa: E731
         clip = moe.ClipGradByGlobalNorm(
             clip_norm=args.grad_clip,
             is_expert_param_func=is_expert_param_fun,