Polish the benchmark scripts and add test for N4C32.

tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_DP1-MP2-PP4-mbs2-acc16-recompute.sh → tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_MP2-PP4-vpp5-mbs2-acc16-recompute.sh

@@ -15,16 +15,15 @@
 
 param="model_name_or_path=facebook/llama-13b "
 param+="per_device_train_batch_size=2 "
-param+="tensor_parallel_degree=2 "
 param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=2 "
 param+="pipeline_parallel_degree=4 "
 param+="virtual_pp_degree=5 "
 param+="sequence_parallel=0 "
 param+="sharding_parallel_degree=1 "
-param+="save_steps=200 "
 param+="sharding=stage1 "
 param+="recompute=1 "
-param+="run_mode=DP1-MP2-PP4-mbs2-acc16-recompute "
+param+="run_mode=MP2-PP4-vpp5-mbs2-acc16-recompute "
 param+="device_num=N1C8 "
 param+="global_batch_size=32 "
 param+="model_item=facebook-llama-13b_seqlen2048_pretrain "

tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_MP4-PP2-mbs1-acc32.sh

@@ -0,0 +1,39 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=4 "
+param+="pipeline_parallel_degree=2 "
+param+="virtual_pp_degree=1 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=1 "
+param+="sharding=stage1 "
+param+="recompute=0 "
+param+="run_mode=MP4-PP2-mbs1-acc32 "
+param+="device_num=N1C8 "
+param+="global_batch_size=32 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_DP1-MP1-PP8-mbs1-acc32-recompute.sh → tests/test_tipc/dygraph/hybrid_parallelism/llama/N1C8/facebook-llama-13b_seqlen2048_pretrain_bs32_bf16_PP8-vpp5-mbs1-acc32-recompute.sh

@@ -15,16 +15,15 @@
 
 param="model_name_or_path=facebook/llama-13b "
 param+="per_device_train_batch_size=1 "
-param+="tensor_parallel_degree=1 "
 param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=1 "
 param+="pipeline_parallel_degree=8 "
 param+="virtual_pp_degree=5 "
 param+="sequence_parallel=0 "
 param+="sharding_parallel_degree=1 "
-param+="save_steps=200 "
 param+="sharding=stage1 "
 param+="recompute=1 "
-param+="run_mode=DP1-MP1-PP8-mbs1-acc32-recompute "
+param+="run_mode=PP8-vpp5-mbs1-acc32-recompute "
 param+="device_num=N1C8 "
 param+="global_batch_size=32 "
 param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
@@ -37,4 +36,4 @@ param+="recompute_use_reentrant=true "
 cd ./tests
 bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
 
-bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N2C16/facebook-llama-13b_seqlen2048_pretrain_bs64_bf16_DP1-MP2-PP4-VPP1-mbs1-acc32-recompute.sh → tests/test_tipc/dygraph/hybrid_parallelism/llama/N2C16/facebook-llama-13b_seqlen2048_pretrain_bs64_bf16_MP2-PP4-SD2-stage1-mbs1-acc32.sh

@@ -14,16 +14,15 @@
 
 param="model_name_or_path=facebook/llama-13b "
 param+="per_device_train_batch_size=1 "
-param+="tensor_parallel_degree=2 "
 param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=2 "
 param+="pipeline_parallel_degree=4 "
 param+="virtual_pp_degree=1 "
 param+="sequence_parallel=0 "
 param+="sharding_parallel_degree=2 "
-param+="save_steps=200 "
 param+="sharding=stage1 "
 param+="recompute=0 "
-param+="run_mode=DP1-MP2-PP4-VPP1-mbs1-acc32-recompute "
+param+="run_mode=MP2-PP4-SD2-stage1-mbs1-acc32 "
 param+="device_num=N2C16 "
 param+="global_batch_size=64 "
 param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
@@ -36,4 +35,4 @@ param+="recompute_use_reentrant=true "
 cd ./tests
 bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
 
-bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N2C16/facebook-llama-13b_seqlen2048_pretrain_bs64_bf16_DP1-MP2-PP4-VPP2-mbs1-acc32-recompute.sh → tests/test_tipc/dygraph/hybrid_parallelism/llama/N2C16/facebook-llama-13b_seqlen2048_pretrain_bs64_bf16_MP2-PP4-SD2-stage1-vpp2-mbs1-acc32.sh

@@ -14,16 +14,15 @@
 
 param="model_name_or_path=facebook/llama-13b "
 param+="per_device_train_batch_size=1 "
-param+="tensor_parallel_degree=2 "
 param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=2 "
 param+="pipeline_parallel_degree=4 "
 param+="virtual_pp_degree=2 "
 param+="sequence_parallel=0 "
 param+="sharding_parallel_degree=2 "
-param+="save_steps=200 "
 param+="sharding=stage1 "
 param+="recompute=0 "
-param+="run_mode=DP1-MP2-PP4-VPP2-mbs1-acc32-recompute "
+param+="run_mode=MP2-PP4-SD2-stage1-vpp2-mbs1-acc32 "
 param+="device_num=N2C16 "
 param+="global_batch_size=64 "
 param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
@@ -36,4 +35,4 @@ param+="recompute_use_reentrant=true "
 cd ./tests
 bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
 
-bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N2C16/facebook-llama-13b_seqlen2048_pretrain_bs64_bf16_PP8-SD2-stage1-mbs1-acc32.sh

@@ -0,0 +1,38 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=1 "
+param+="pipeline_parallel_degree=8 "
+param+="virtual_pp_degree=1 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=2 "
+param+="sharding=stage1 "
+param+="recompute=0 "
+param+="run_mode=PP8-SD2-stage1-mbs1-acc32 "
+param+="device_num=N2C16 "
+param+="global_batch_size=64 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N4C32/facebook-llama-13b_seqlen2048_pretrain_bs128_bf16_MP2-PP4-SD4-stage1-mbs1-acc32.sh

@@ -0,0 +1,39 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=2 "
+param+="pipeline_parallel_degree=4 "
+param+="virtual_pp_degree=1 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=4 "
+param+="sharding=stage1 "
+param+="recompute=0 "
+param+="run_mode=MP2-PP4-SD4-stage1-mbs1-acc32 "
+param+="device_num=N4C32 "
+param+="global_batch_size=128 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="pipeline_parallel_config=enable_sharding_comm_overlap "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/N4C32/facebook-llama-13b_seqlen2048_pretrain_bs128_bf16_PP8-SD4-stage1-mbs1-acc32.sh

@@ -0,0 +1,38 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_name_or_path=facebook/llama-13b "
+param+="per_device_train_batch_size=1 "
+param+="data_parallel_degree=1 "
+param+="tensor_parallel_degree=1 "
+param+="pipeline_parallel_degree=8 "
+param+="virtual_pp_degree=1 "
+param+="sequence_parallel=0 "
+param+="sharding_parallel_degree=4 "
+param+="sharding=stage1 "
+param+="recompute=0 "
+param+="run_mode=PP8-SD4-stage1-mbs1-acc32 "
+param+="device_num=N4C32 "
+param+="global_batch_size=128 "
+param+="model_item=facebook-llama-13b_seqlen2048_pretrain "
+param+="max_steps=150 "
+param+="gradient_accumulation_steps=32 "
+param+="pp_recompute_interval=1 "
+param+="tensor_parallel_config=enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add "
+param+="recompute_use_reentrant=true "
+
+cd ./tests
+bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh"

tests/test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Test training benchmark for a model.
-# Usage：bash benchmark/run_benchmark.sh ${model_name_or_path} ${per_device_train_batch_size} ${tensor_parallel_degree} ${pipeline_parallel_degree} ${virtual_pp_degree} ${sequence_parallel} ${sharding_parallel_degree} ${save_steps} ${sharding} ${recompute} ${run_mode} ${device_num}
+# Usage：bash benchmark/run_benchmark.sh ${model_name_or_path} ${per_device_train_batch_size} ${tensor_parallel_degree} ${pipeline_parallel_degree} ${virtual_pp_degree} ${sequence_parallel} ${sharding_parallel_degree} ${sharding} ${recompute} ${run_mode} ${device_num}
 function _set_params(){
     model_name_or_path=${model_name_or_path:-"facebook/llama-13b"}
     per_device_train_batch_size=${per_device_train_batch_size:-1}
@@ -25,7 +25,6 @@ function _set_params(){
     virtual_pp_degree=${virtual_pp_degree:-2}
     sequence_parallel=${sequence_parallel:-0}
     sharding_parallel_degree=${sharding_parallel_degree:-2}
-    save_steps=${save_steps:-200}
     sharding=${sharding:-"stage1"}
     recompute=${recompute:-1}
     run_mode=${run_mode:-"DP1-MP2-PP4-mbs1-acc32-recompute"}
@@ -36,6 +35,7 @@ function _set_params(){
     gradient_accumulation_steps=${gradient_accumulation_steps:-32}
     pp_recompute_interval=${pp_recompute_interval:-1}
     tensor_parallel_config=${tensor_parallel_config:-"enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add"}
+    pipeline_parallel_config=${pipeline_parallel_config:-""}
     recompute_use_reentrant=${recompute_use_reentrant:-"true"}
 
     base_batch_size=${global_batch_size}
@@ -92,6 +92,12 @@ function _train(){
         use_fp16_cmd="--use_amp true"
     fi
 
+    if [ "${pipeline_parallel_config}" != "" ]; then
+        pipeline_parallel_config_args="--pipeline_parallel_config ${pipeline_parallel_config}"
+    else
+        pipeline_parallel_config_args=""
+    fi
+
     use_pure_fp16=False
     train_cmd="--model_type llama \
     --model_name_or_path ${model_name_or_path} \
@@ -130,7 +136,7 @@ function _train(){
     --enable_linear_fused_grad_add true \
     --fuse_attention_qkv true \
     --fuse_attention_ffn true \
-    --tensor_parallel_config ${tensor_parallel_config} \
+    --tensor_parallel_config ${tensor_parallel_config} ${pipeline_parallel_config_args} \
     --recompute ${recompute} \
     --recompute_use_reentrant ${recompute_use_reentrant} \
     --data_cache ./data_cache"
@@ -142,33 +148,28 @@ function _train(){
         PADDLE_RANK_OPTION=""
     fi
     # 以下为通用执行命令，无特殊可不用修改
-    if [ "N1C2" = ${device_num} ]; then
-        # sharding case
-        echo "run run_mode: DP1-MP1-PP1 device_num: N1C2"
+    case ${device_num} in
+    N1C1) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
+        train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0 ${PADDLE_RANK_OPTION}\
+            run_pretrain.py ${train_cmd}"
+        workerlog_id=0
+        ;;
+    N1C2) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
         train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0,1 ${PADDLE_RANK_OPTION}\
               run_pretrain.py ${train_cmd}" 
         workerlog_id=0
-    else
-        # hybrid_parallelism case
-        case ${run_mode} in
-        DP1-MP1-PP1) echo "run run_mode: DP1-MP1-PP1"
-            train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0 ${PADDLE_RANK_OPTION}\
-                run_pretrain.py ${train_cmd}"
-            workerlog_id=0
-            ;;
-        DP1-MP1-PP4|DP1-MP4-PP1) echo "run run_mode: ${run_mode}"
-            train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0,1,2,3 ${PADDLE_RANK_OPTION}\
-                run_pretrain.py ${train_cmd}"
-            workerlog_id=0
-            ;;
-        DP1-MP2-PP4-mbs2-acc16-recompute|DP1-MP1-PP8-mbs1-acc32-recompute|DP1-MP2-PP4-VPP2-mbs1-acc32-recompute|DP1-MP2-PP4-VPP1-mbs1-acc32-recompute) echo "run run_mode: ${run_mode}"
-            train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0,1,2,3,4,5,6,7 ${PADDLE_RANK_OPTION}\
-                run_pretrain.py ${train_cmd}"
-            workerlog_id=0
-            ;;
-        *) echo "choose run_mode "; exit 1;
-        esac
-    fi
+        ;;
+    N1C4) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
+        train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0,1,2,3 ${PADDLE_RANK_OPTION}\
+            run_pretrain.py ${train_cmd}"
+        workerlog_id=0
+        ;;
+    *) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
+        train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=0,1,2,3,4,5,6,7 ${PADDLE_RANK_OPTION}\
+            run_pretrain.py ${train_cmd}"
+        workerlog_id=0
+        ;;
+    esac
     cd ../llm/llama
     echo "train_cmd: ${train_cmd}  log_file: ${log_file}"
     python -c "import paddlenlp"