add pir llama dp ci test.

Author: winter-wang

scripts/distribute/ci_case_auto.sh

@@ -62,6 +62,8 @@ function llama_case_list_auto() {
     llama_dygraph_auto_bs8_fp16_DP2-MP2-PP2
     llama_dy2st_auto_bs4_bf16_DP1-MP1-PP4-SD2
 
+    llama_dy2st_pir_auto_bs1_fp32_DP2-MP1-PP1
+
     llama_static_auto_recompute_bs8_fp32_DP1-MP1-PP1
     llama_static_auto_recompute_bs16_fp32_DP2-MP1-PP1
     llama_static_auto_recompute_bs16_fp32_DP2-MP2-PP1
@@ -1751,6 +1753,68 @@ function llama_dy2st_auto_bs4_bf16_DP1-MP1-PP4-SD2() {
     echo "=========== $FUNCNAME run  end ==========="
 }
 
+function llama_dy2st_pir_auto_bs1_fp32_DP2-MP1-PP1() {
+    echo "=========== $FUNCNAME run begin ==========="
+    export PYTHONPATH=$root_path/:$PYTHONPATH
+    export FLAGS_call_stack_level=3
+    export NVIDIA_TF32_OVERRIDE=0
+
+    task_name="llama_dy2st_pir_auto_bs1_fp32_dp2mp1pp1"
+    case_out_dir="output/$task_name"
+    case_log_dir="output/$task_name""_log"
+    rm -rf $case_out_dir
+    rm -rf $case_log_dir
+
+    python -u -m paddle.distributed.launch --gpus "0,1" --log_dir $case_log_dir run_pretrain_auto.py \
+        --model_type "llama" \
+        --model_name_or_path "facebook/llama-7b" \
+        --tokenizer_name_or_path "facebook/llama-7b" \
+        --input_dir "./data" \
+        --output_dir $case_out_dir \
+        --split 949,50,1 \
+        --max_seq_length 2048 \
+        --per_device_train_batch_size 1 \
+        --per_device_eval_batch_size 0 \
+        --gradient_accumulation_steps 1 \
+        --use_flash_attention 0 \
+        --use_fused_rms_norm 0 \
+        --fp16 0 \
+        --fp16_opt_level "O2"  \
+        --scale_loss 1024 \
+        --pipeline_parallel_degree 1 \
+        --tensor_parallel_degree 1 \
+        --sharding_parallel_degree 1 \
+        --learning_rate 0.0001 \
+        --min_learning_rate 0.00001 \
+        --max_steps 10 \
+        --save_steps 5000000 \
+        --weight_decay 0.01 \
+        --warmup_ratio 0.01 \
+        --logging_steps 1\
+        --dataloader_num_workers 1 \
+        --sharding "" \
+        --eval_steps 1000000 \
+        --disable_tqdm true \
+        --continue_training 0\
+        --recompute 0 \
+        --do_train \
+        --do_eval 0\
+        --device "gpu" \
+        --data_impl "mmap" \
+        --enable_auto_parallel 1 \
+        --max_grad_norm 1.0 \
+        --to_static $to_static \
+        --num_hidden_layers 2 \
+        --hidden_size 1024 \
+        --intermediate_size 4096 \
+        >>${log_path}/$FUNCNAME 2>&1
+    loss=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
+    ips=-1
+    mem=-1
+    echo "result: loss=$loss ips=$ips mem=$mem"
+    echo "=========== $FUNCNAME run  end ==========="
+}
+
 function llm_gpt_dygraph_auto_bs8_fp32_DP2() {
     echo "=========== $FUNCNAME run begin ==========="
     export PYTHONPATH=$root_path/:$PYTHONPATH