[TIPC]Polish dy2st tipc shell (#4155)

* Polish dy2st tipc shell

* fix errors.

Co-authored-by: xiongkun <[email protected]>
Co-authored-by: zhang junjun <[email protected]>

Author: 3 people

tests/test_tipc/benchmark_train.sh

@@ -113,10 +113,6 @@ to_static=""
 if [[ $PARAMS =~ "dynamicTostatic" ]] ;then
    to_static="d2sT_"
    sed -i 's/trainer:norm_train/trainer:to_static_train/g' $FILENAME
-   # clear PARAM contents
-   if [ $PARAMS = "to_static" ] ;then
-    PARAMS=""
-   fi
 fi
 
 
@@ -135,7 +131,7 @@ export frame_version=${str_tmp%%.post*}
 export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
 
 # 获取benchmark_params所在的行数
-line_num=`grep -n "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+line_num=`grep -n -w "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
 # for train log parser
 batch_size=$(func_parser_value "${lines[line_num]}")
 line_num=`expr $line_num + 1`
@@ -189,6 +185,13 @@ if  [ ! -n "$PARAMS" ] ;then
     fp_items_list=(${fp_items})
     device_num_list=(N1C4)
     run_mode="DP"
+elif [[ ${PARAMS} = "dynamicTostatic" ]] ;then
+    IFS="|"
+    model_type=$PARAMS
+    batch_size_list=(${batch_size})
+    fp_items_list=(${fp_items})
+    device_num_list=(N1C4)
+    run_mode="DP"
 else
     # parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_mode}_${device_num}
     IFS="_"

tests/test_tipc/configs/bert/base/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:../examples/language_model/bert/run_pretrain.py --max_predictions_per
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:--to_static True
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -49,6 +49,8 @@ null:null
 null:null
 null:null
 null:null
+===========================to_static_train_benchmark_params===========================
+to_static_train:--to_static True
 ===========================train_benchmark_params==========================
 batch_size:32|64|96
 fp_items:fp32|fp16

tests/test_tipc/configs/bert/large/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:../examples/language_model/bert/run_pretrain.py --max_predictions_per
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:--to_static True
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -49,6 +49,8 @@ null:null
 null:null
 null:null
 null:null
+===========================to_static_train_benchmark_params===========================
+to_static_train:--to_static True
 ===========================train_benchmark_params==========================
 batch_size:4
 fp_items:fp32|fp16

tests/test_tipc/configs/gpt2/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:../examples/language_model/gpt/run_pretrain.py --model_type="gpt" --m
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:--to_static
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -49,6 +49,8 @@ null:null
 null:null
 null:null
 null:null
+===========================to_static_train_benchmark_params===========================
+to_static_train:--to_static
 ===========================train_benchmark_params==========================
 batch_size:8
 fp_items:fp32|fp16

tests/test_tipc/configs/gpt3/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:../examples/language_model/gpt-3/dygraph/run_pretrain.py --model_type
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:--to_static
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -49,6 +49,8 @@ null:null
 null:null
 null:null
 null:null
+===========================to_static_train_benchmark_params===========================
+to_static_train:--to_static
 ===========================train_benchmark_params==========================
 batch_size:8
 fp_items:fp16|fp32

tests/test_tipc/configs/transformer/base/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:../examples/machine_translation/transformer/train.py --config ../exam
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:--to_static
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -49,6 +49,8 @@ null:null
 null:null
 null:null
 null:null
+===========================to_static_train_benchmark_params===========================
+to_static_train:--to_static
 ===========================train_benchmark_params==========================
 batch_size:4096
 fp_items:fp32|fp16

tests/test_tipc/configs/transformer/big/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:../examples/machine_translation/transformer/train.py --config ../exam
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:--to_static
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -49,6 +49,8 @@ null:null
 null:null
 null:null
 null:null
+===========================to_static_train_benchmark_params===========================
+to_static_train:--to_static
 ===========================train_benchmark_params==========================
 batch_size:5120|4096
 fp_items:fp32|fp16

tests/test_tipc/test_train_dy2static_python.sh

@@ -64,7 +64,7 @@ do
     # start dy2static train
     dy2static_output=$LOG_PATH/${config_name}_python_train_infer_dy2static_output.txt
     dy2static_loss=$LOG_PATH/${config_name}_dy2static_loss.txt
-    sed -i '15ctrainer:to_static_train' ${FILENAME}
+    sed -i '16s/$/ --to_static/g' ${FILENAME}
     cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} $MODE >$dy2static_output 2>&1"
     echo $cmd
     eval $cmd

tests/test_tipc/test_train_inference_python.sh

@@ -20,7 +20,7 @@ FILENAME=$1
 # MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer' 'whole_train_whole_infer', 'whole_infer', 'klquant_whole_infer']
 MODE=$2
 
-dataline=$(awk 'NR==1, NR==51{print}'  $FILENAME)
+dataline=$(cat ${FILENAME})
 
 # parser params
 IFS=$'\n'
@@ -55,8 +55,8 @@ fpgm_key=$(func_parser_key "${lines[17]}")
 fpgm_trainer=$(func_parser_value "${lines[17]}")
 distill_key=$(func_parser_key "${lines[18]}")
 distill_trainer=$(func_parser_value "${lines[18]}")
-to_static_key=$(func_parser_key "${lines[19]}")
-to_static_trainer=$(func_parser_value "${lines[19]}")
+trainer_key1=$(func_parser_key "${lines[19]}")
+trainer_value1=$(func_parser_value "${lines[19]}")
 trainer_key2=$(func_parser_key "${lines[20]}")
 trainer_value2=$(func_parser_value "${lines[20]}")
 
@@ -103,6 +103,10 @@ benchmark_value=$(func_parser_value "${lines[49]}")
 infer_key1=$(func_parser_key "${lines[50]}")
 infer_value1=$(func_parser_value "${lines[50]}")
 
+line_num=`grep -n -w "to_static_train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+to_static_key=$(func_parser_key "${lines[line_num]}")
+to_static_trainer=$(func_parser_value "${lines[line_num]}")
+
 # parser klquant_infer
 if [ ${MODE} = "klquant_whole_infer" ]; then
     dataline=$(awk 'NR==1, NR==17{print}'  $FILENAME)
@@ -312,15 +316,18 @@ else
                 elif [ ${trainer} = "${distill_key}" ]; then
                     run_train=${distill_trainer}
                     run_export=${distill_export}
+                elif [[ ${trainer} = ${trainer_key1} ]]; then
+                    run_train=${trainer_value1}
+                    run_export=${export_value1}
+                elif [[ ${trainer} = ${trainer_key2} ]]; then
+                    run_train=${trainer_value2}
+                    run_export=${export_value2}
                 # In case of @to_static, we re-used norm_traier,
                 # but append "--to_static" for config
                 # to trigger "apply_to_static" logic in 'train.py'
                 elif [ ${trainer} = "${to_static_key}" ]; then
                     run_train="${norm_trainer}  ${to_static_trainer}"
                     run_export=${norm_export}
-                elif [[ ${trainer} = ${trainer_key2} ]]; then
-                    run_train=${trainer_value2}
-                    run_export=${export_value2}
                 else
                     run_train=${norm_trainer}
                     run_export=${norm_export}