sp default disenable

Author: JZ-LIANG

model_zoo/gpt-3/ppfleetx/configs/nlp/gpt/auto/pretrain_gpt_base.yaml

@@ -35,7 +35,7 @@ Model:
   scale_qk_by_layer_num: True
   fused_softmax_with_triangular: True
   use_flash_attn: False
-
+  sequence_parallel: False # TODO make sequence_parallel as an independent parallel and be set id Distributed
 
 Data:
   Train:

model_zoo/gpt-3/ppfleetx/utils/auto_config.py

@@ -16,6 +16,8 @@
 import os
 import sys
 
+from sympy import sequence
+
 import paddle
 import paddle.distributed as dist
 import paddle.distributed.auto_parallel as auto
@@ -42,6 +44,15 @@ def process_dist_configs(config):
     mp_degree = configs.setdefault("mp_degree", 1)
     pp_degree = configs.setdefault("pp_degree", 1)
 
+    # disenable sequence parallel is mp_degree < 2.
+    sequence_parallel = config["Model"]["sequence_parallel"]
+    if mp_degree < 2 and sequence_parallel:
+        config["Model"]["sequence_parallel"] = False
+        logger.warning(
+            "sequence_parallel is turn off since mp_degree < 2."
+        )
+
+
     # sharding default
     sharding_config = configs["sharding"]
     sharding_degree = sharding_config.setdefault("sharding_degree", 1)