add test

Author: waliwali777

scripts/distribute/ci_case_auto.sh

@@ -94,6 +94,7 @@ function llama_case_list_auto() {
         llama_dy2st_auto_bs4_bf16_DP1-MP1-PP4-SD2-VPP3_split_bw
         llama_dy2st_auto_bs4_bf16_DP1-MP1-PP4-SD2
         llama_align_dygraph_dy2st_auto_bs2_bf16_DP2-MP1-PP1
+        llama_pir_auto_recompute_DP2_MP2_PP2
         llama_pir_auto_fuse_ffn_attention_qkv_MP2
         llama_convert_hybrid_ckpt_to_auto_parallel_bs2_fp32_DP2-MP1-PP1
         llama_align_dygraph_dy2st_pir_auto_bs2_bf16_DP2-MP2-PP1-SP
@@ -743,6 +744,107 @@ function llama_align_dygraph_dy2st_pir_auto_bs2_bf16_DP2-MP2-PP1-SP() {
     echo "=========== $FUNCNAME run  end ==========="
 }
 
+function llama_pir_auto_recompute_DP2_MP2_PP2(){
+    echo "=========== $FUNCNAME run begin ==========="
+    export PYTHONPATH=$root_path/:$PYTHONPATH
+    export PYTHONPATH=/paddle/Paddle/build_gpu/python/:$PYTHONPATH
+    export FLAGS_call_stack_level=3
+    export FLAGS_enable_pir_api=1
+    export FLAGS_dynamic_static_unified_comm=1
+    export FLAGS_enable_auto_parallel_align_mode=1
+
+    export NVIDIA_TF32_OVERRIDE=0
+    export FLAGS_cudnn_deterministic=1
+    export FLAGS_embedding_deterministic=1
+
+    task_name="llama_pir_auto_recompute_DP2_MP2_PP2"
+    case_out_dir="output/$task_name"
+    case_log_dir="output/$task_name""_log"
+    
+    loss1=0
+    loss2=0
+
+    for use_recompute in "0" "1"; do
+        rm -rf $case_out_dir
+        rm -rf $case_log_dir
+        python -u -m paddle.distributed.launch \
+            --gpus "0,1,2,3,4,5,6,7" \
+            --log_dir $case_log_dir \
+            run_pretrain_auto.py \
+            --model_type "llama" \
+            --model_name_or_path "facebook/llama-7b" \
+            --tokenizer_name_or_path "facebook/llama-7b" \
+            --input_dir "./data" \
+            --output_dir $case_out_dir \
+            --split 949,50,1 \
+            --to_static true \
+            --pipeline_parallel_degree 2 \
+            --tensor_parallel_degree 2 \
+            --virtual_pp_degree 1 \
+            --pipeline_schedule_mode "VPP" \
+            --weight_decay 0.01 \
+            --warmup_ratio 0.01 \
+            --max_grad_norm 0.0 \
+            --learning_rate 3e-05 \
+            --min_learning_rate 3e-06 \
+            --max_steps 10 \
+            --logging_steps 10 \
+            --eval_steps 10000 \
+            --save_steps 1000 \
+            --continue_training 0 \
+            --do_train true \
+            --do_eval false \
+            --do_predict false \
+            --disable_tqdm true \
+            --save_total_limit 2 \
+            --device gpu \
+            --dataloader_num_workers 4 \
+            --distributed_dataloader 0 \
+            --enable_auto_parallel 1 \
+            --per_device_train_batch_size 1 \
+            --gradient_accumulation_steps 1 \
+            --per_device_eval_batch_size 1 \
+            --recompute ${use_recompute} \
+            --recompute_use_reentrant true \
+            --recompute_granularity full \
+            --pp_recompute_interval 0 \
+            --bf16 true \
+            --fp16_opt_level "O2"  \
+            --amp_master_grad true \
+            --fuse_attention_ffn true \
+            --fuse_attention_qkv true \
+            --use_flash_attention false \
+            --use_fused_rope true \
+            --use_fused_rms_norm false \
+            --max_seq_length 4096 \
+            --sequence_parallel false \
+            --sharding "stage1" \
+            --data_parallel_config "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate " \
+            --sharding_parallel_config "enable_stage1_overlap" \
+            --tensor_parallel_config "enable_mp_async_allreduce" \
+            --pipeline_parallel_config "enable_send_recv_overlap" \
+            --auto_parallel_resume_form_hybrid_parallel true \
+            --num_hidden_layers 4 \
+            >>${log_path}/$FUNCNAME 2>&1
+            loss=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
+            loss_md5=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'loss_md5: ' '{print $2}' | awk -F ',' '{print $1}'`
+            ips=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'interval_tokens_per_second_per_device: ' '{print $2}' | awk -F ',' '{print $1}'`
+            mem=`cat $case_log_dir/workerlog.0 | grep 'global_step: 10' | awk -F 'max_memory_reserved: ' '{print $2}' | awk -F ',' '{print $1}'`
+            echo "result: loss=$loss loss_md5=$loss_md5 ips=$ips mem=$mem"
+            if [ $use_recompute -eq 0 ];then
+                loss1=($loss)
+            else
+                loss2=($loss)
+            fi
+    done
+    ips=-1
+    mem=-1
+    ips_base=-1
+    mem_base=-1
+    check_result $FUNCNAME ${loss1} ${loss2} ${ips_base} ${ips} ${mem_base} ${mem}
+    echo "=========== $FUNCNAME run  end ==========="
+}
+
 function llama_pir_auto_fuse_ffn_attention_qkv_MP2() {
     echo "=========== $FUNCNAME run begin ==========="
     export PYTHONPATH=$root_path/:$PYTHONPATH