[Distributed]Add loss nan/inf checker (#8943)

* add checker

* fix fp16

Author: ForFishes

paddlenlp/trainer/trainer.py

@@ -992,6 +992,10 @@ def _inner_training_loop(
                 else:
                     tr_loss_step = self.training_step(model, inputs)
 
+                if not args.fp16:
+                    if not paddle.isfinite(tr_loss_step).all().item():
+                        raise ValueError(f"Loss contains inf or nan values at rank {paddle.distributed.get_rank()}")
+
                 tr_loss += tr_loss_step
 
                 def fused_allreduce_gradients_no_sync(paramlist, hcg):