[AutoParallel] Support qwen for auto_parallel (#8312)

* run 4096

* support 3d semi auto run of qwen model

* [AutoParallel] Add qwen auto parallel.

* Delete useless files.

* Fix code style problem.

* Fix problems.

* Polish code.

* Add CI-cases.

* Add bf16 ci testcase.

* Fix some problems.

---------

Co-authored-by: pangengzheng <[email protected]>
Co-authored-by: pangengzheng <pangengzheng.baidu.com>

Author: GhostScreaming

llm/qwen/auto_parallel/pretrain_argument_auto_dp2tp2pp2.json

@@ -0,0 +1,45 @@
+{
+    "model_name_or_path": "qwen/qwen-14b",
+    "tokenizer_name_or_path": "qwen/qwen-14b",
+    "input_dir": "./data",
+    "output_dir": "./checkpoints/qwen_pretrain_ckpts",
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 2,
+    "per_device_eval_batch_size": 16,
+    "data_parallel_degree": 2,
+    "tensor_parallel_degree": 2,
+    "pipeline_parallel_degree": 2,
+    "virtual_pp_degree": 1,
+    "sequence_parallel": 0,   
+    "use_flash_attention": false,
+    "use_fused_rms_norm": false,
+    "use_fused_rope": false,
+    "max_seq_length": 4096,
+    "learning_rate": 3e-05,
+    "min_learning_rate": 3e-06,
+    "scale_loss": 1024,
+    "warmup_steps": 30,
+    "logging_steps": 1,
+    "max_steps": 10000,
+    "save_steps": 1000,
+    "eval_steps": 10000,
+    "weight_decay": 0.01,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "warmup_ratio": 0.01,
+    "max_grad_norm": 1.0,
+    "dataloader_num_workers": 1,
+    "continue_training": 0,
+    "do_train": true,
+    "do_eval": true,
+    "do_predict": true,
+    "disable_tqdm": true,
+    "recompute": true,
+    "recompute_granularity": "core_attn",
+    "recompute_use_reentrant": true,
+    "distributed_dataloader": 0,
+    "save_total_limit": 2,
+    "parallel_mode": "auto",
+    "enable_auto_parallel": 1,
+    "to_static": 0
+  }