[CI] update scripts (#9547)

Author: Liujie0926

paddlenlp/experimental/autonlp/requirements.txt

@@ -1,4 +1,4 @@
 protobuf==3.20.2 
-pydantic==1.10.11
+pydantic
 ray[tune]==2.5.1
 hyperopt>=0.2.5

requirements-dev.txt

@@ -29,4 +29,4 @@ data
 wget
 huggingface_hub>=0.19.2
 tiktoken
-tokenizers
+tokenizers>=0.21,<0.22

requirements.txt

@@ -27,4 +27,4 @@ jinja2
 regex
 numpy<=1.26.4
 tiktoken
-tokenizers<0.21
+tokenizers>=0.21,<0.22

scripts/distribute/run_ci.sh

@@ -205,6 +205,33 @@ function execute_func_list(){
     echo -e "\033[31m $(printf '\t')  exit 250 tests(intermittent issue) :  $exit_250_count \033"
 }
 
+function clean_file(){
+    target_path=$1
+    matching_data_dirs=$(find "$target_path" -maxdepth 1 -type d -name "*data*")
+    if [ -n "$matching_data_dirs" ]; then
+        echo "cleaning data dirs:"
+        echo $matching_data_dirs        
+        for dir in $matching_data_dirs; do
+            rm -rf "$dir"
+            echo "deleted $dir"
+        done
+    else
+        echo "$target_path no data dirs found"
+    fi
+
+    matching_output_dirs=$(find "$target_path" -maxdepth 1 -type d -name "*output*")
+    if [ -n "$matching_output_dirs" ]; then
+        echo "cleaning output dirs:"
+        echo $matching_output_dirs
+        for dir in $matching_output_dirs; do
+            rm -rf "$dir"
+            echo "deleted $dir"
+        done
+    else
+        echo "$target_path no output dirs found"
+    fi
+}
+
 ####################################
 get_diff_TO_case # 获取待执行case列表
 case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${case_list[*]}))  # 去重并将结果存储回原列表
@@ -231,6 +258,7 @@ if [[ ${#case_list[*]} -ne 0 ]];then
         execute_func_list $cmd llama_auto
         export FLAGS_download_data="llama ""$FLAGS_download_data"
         let case_num++
+        clean_file $nlp_dir/llm/auto_parallel/llama
     fi
     if [[ $(contain_case gpt-3_auto ${case_list[@]}; echo $?) -eq 1 ]];then
         echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: gpt-3_auto \033"
@@ -240,6 +268,7 @@ if [[ ${#case_list[*]} -ne 0 ]];then
         export FLAGS_install_deps=1
         export FLAGS_download_data="gpt ""$FLAGS_download_data"
         let case_num++        
+        clean_file $nlp_dir/llm/auto_parallel/gpt-3
     fi
     if [[ $(contain_case gpt-3_dygraph ${case_list[@]}; echo $?) -eq 1 ]];then
         echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: gpt-3_dygraph \033"
@@ -249,6 +278,7 @@ if [[ ${#case_list[*]} -ne 0 ]];then
         export FLAGS_install_deps=1
         export FLAGS_download_data="gpt ""$FLAGS_download_data"
         let case_num++
+        clean_file $nlp_dir/slm/model_zoo/gpt-3
     fi
     echo -e "\033[31m ---- end run case  \033"
 

scripts/regression/run_ci.sh

@@ -66,8 +66,9 @@ install_paddle(){
     echo -e "\033[35m ---- Install paddlepaddle-gpu  \033[0m"
     python -m pip install --user -r scripts/regression/requirements_ci.txt
     python -m pip uninstall paddlepaddle -y
-    python -m pip install --user ${paddle};
-    python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
+    python -m pip install pillow -y
+    python -m pip install --user ${paddle} --no-cache-dir;
+    python -c "import paddle;print('paddle')print(paddle.__version__);print(paddle.version.show())" >> ${log_path}/commit_info.txt
     python -c 'from visualdl import LogWriter'
 }
 ####################################
@@ -237,6 +238,7 @@ if [[ ${#P0case_list[*]} -ne 0 ]] || [[ ${#APIcase_list[*]} -ne 0 ]];then
         echo "instal_nlp_pr"
         python -m pip install  dist/p****.whl
     fi
+    python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)" >> ${log_path}/commit_info.txt
     python -m pip list
     echo -e "\033[35m =======CI Check P0case========= \033[0m"
     echo -e "\033[35m ---- P0case_list length: ${#P0case_list[*]}, cases: ${P0case_list[*]} \033[0m"

scripts/unit_test/ci_unit.sh

@@ -31,10 +31,13 @@ install_requirements() {
     python -m pip install -r tests/requirements.txt
     python -m pip install -r paddlenlp/experimental/autonlp/requirements.txt 
     python -m pip uninstall paddlepaddle paddlepaddle_gpu -y
+    python -m pip install pillow -y
     python -m pip install --no-cache-dir ${paddle}
+    python -c "import paddle;print('paddle')print(paddle.__version__);print(paddle.version.show())" >> ${log_path}/commit_info.txt
 
     python setup.py bdist_wheel > /dev/null
     python -m pip install  dist/p****.whl
+    python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)" >> ${log_path}/commit_info.txt
     cd csrc/
     python setup_cuda.py install
     cd ../

tests/requirements.txt

@@ -5,7 +5,7 @@ torch>=1.5
 transformers
 fast_dataindex
 sacremoses
-pydantic==1.10.9
+pydantic
 modelscope
 hyperopt
 h5py