Fix dataset with empty char. (#8469)

* fix dataset with empty char.

* revert file links.

Author: ZHUI

docs/llm/pretraining/data/OpenWebText2.md

@@ -14,7 +14,7 @@
 ```shell
 # wget https://mystic.the-eye.eu/public/AI/pile_preliminary_components/openwebtext2.jsonl.zst.tar
 wget https://paddlenlp.bj.bcebos.com/models/transformers/gpt/openwebtext2.jsonl.zst.tar
-tar -xvf openwebtext2.json.zst.tar -C  /path/to/openwebtext
+tar -xvf openwebtext2.jsonl.zst.tar -C  /path/to/openwebtext
 ```
 
 ## Llama训练数据制作

llm/README.md

@@ -61,20 +61,20 @@ PaddleNLP将飞桨4D并行策略加入到Trainer API中， 用户只需修改Tra
 为了方便用户运行测试本模型，本项目提供了处理好的100k条doc的训练样本：
 ```shell
 # llama 模型数据下载
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
 # gpt 模型数据下载
-# wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy
-# wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_idx.npz
+# wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt2_openwebtext_100k.bin
+# wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt2_openwebtext_100k.idx
 ```
 
 将所有预处理得到的文件统一放入一个文件夹中，以备训练使用：
 
 ```
 mkdir data
-mv llama_openwebtext_100k_ids.npy ./data
-mv llama_openwebtext_100k_idx.npz ./data
+mv llama_openwebtext_100k.bin ./data
+mv llama_openwebtext_100k.idx ./data
 ```
 
 ```shell

llm/docs/pretrain.rst

@@ -44,8 +44,8 @@ git clone 代码到本地，即可开始。
 .. code-block:: bash
 
     # llama 模型数据下载
-    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+    wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
     # gpt 模型数据下载
     # wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy
@@ -57,8 +57,8 @@ git clone 代码到本地，即可开始。
 .. code-block:: bash
 
     mkdir data
-    mv llama_openwebtext_100k_ids.npy ./data
-    mv llama_openwebtext_100k_idx.npz ./data
+    mv llama_openwebtext_100k.bin ./data
+    mv llama_openwebtext_100k.idx ./data
 
 
 

model_zoo/ernie-1.0/preprocess/create_pretraining_data.py

@@ -103,7 +103,7 @@ def get_args():
     group.add_argument("--append_eos", action="store_true", help="Append an <eos> token to the end of a document.")
     group.add_argument("--log_interval", type=int, default=100, help="Interval between progress updates")
     group.add_argument("--workers", type=int, default=1, help="Number of worker processes to launch")
-    group.add_argument("--max_doc_num", type=int, default=sys.maxsize, help="Number of worker processes to launch")
+    group.add_argument("--max_doc_num", type=int, default=sys.maxsize, help="Stop when reach max_doc_num.")
     group.add_argument(
         "--max_repeated_len", type=int, default=100, help="The maximum length of the repeated characters to keep"
     )

tests/llm/test_pretrain.py

@@ -59,8 +59,8 @@ def test_pretrain(self):
             del sys.modules["run_pretrain"]
 
         # Run pretrain
-        URL = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy"
-        URL2 = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz"
+        URL = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin"
+        URL2 = "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx"
         get_path_from_url(URL, root_dir=self.dataset_dir)
         get_path_from_url(URL2, root_dir=self.dataset_dir)
 

tests/test_tipc/auto_tuner/llama_pretrain/benchmark_common/prepare.sh

@@ -24,11 +24,11 @@ cd ../../../llm/llama
 python -m pip install tool_helpers
 
 rm -rf data && mkdir data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
+wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
 
-mv llama_openwebtext_100k_ids.npy ./data
-mv llama_openwebtext_100k_idx.npz ./data
+mv llama_openwebtext_100k.bin ./data
+mv llama_openwebtext_100k.idx ./data
 
 # mv autoconfig
 rm -rf autoconfig

tests/trainer/test_unified_checkpoint.py

@@ -186,8 +186,8 @@ def setUp(self):
         os.environ.update(environment_variables)
 
         files = [
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy",
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx",
         ]
         self.prepare_inputs_data(pretrain_arguments["input_dir"], files)
 
@@ -651,8 +651,8 @@ def setUp(self):
         os.environ.update(environment_variables)
 
         files = [
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy",
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx",
         ]
         self.prepare_inputs_data(pretrain_arguments["input_dir"], files)
 
@@ -693,8 +693,8 @@ def setUp(self):
         os.environ.update(environment_variables)
 
         files = [
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_ids.npy",
-            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k_idx.npz",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin",
+            "https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx",
         ]
         self.prepare_inputs_data(pretrain_arguments["input_dir"], files)