Better message when data files is empty (#586)

* better error message for empty data_files

* allow writer to write empty tables

* raise ValueError instead

Author: lhoestq

datasets/csv/csv.py

@@ -56,6 +56,8 @@ def _info(self):
     def _split_generators(self, dl_manager):
         """ We handle string, list and dicts in datafiles
         """
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
         data_files = dl_manager.download_and_extract(self.config.data_files)
         if isinstance(data_files, (str, list, tuple)):
             files = data_files

datasets/json/json.py

@@ -43,6 +43,8 @@ def _info(self):
     def _split_generators(self, dl_manager):
         """ We handle string, list and dicts in datafiles
         """
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
         data_files = dl_manager.download_and_extract(self.config.data_files)
         if isinstance(data_files, (str, list, tuple)):
             files = data_files

datasets/pandas/pandas.py

@@ -13,6 +13,8 @@ def _info(self):
     def _split_generators(self, dl_manager):
         """ We handle string, list and dicts in datafiles
         """
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
         data_files = dl_manager.download_and_extract(self.config.data_files)
         if isinstance(data_files, (str, list, tuple)):
             files = data_files

datasets/text/text.py

@@ -75,6 +75,8 @@ def _split_generators(self, dl_manager):
             If str or List[str], then the dataset returns only the 'train' split.
             If dict, then keys should be from the `nlp.Split` enum.
         """
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
         data_files = dl_manager.download_and_extract(self.config.data_files)
         if isinstance(data_files, (str, list, tuple)):
             files = data_files

src/nlp/arrow_reader.py

@@ -288,7 +288,8 @@ def _get_dataset_from_filename(self, filename_skip_take):
         mmap = pa.memory_map(filename)
         f = pa.ipc.open_stream(mmap)
         pa_table = f.read_all()
-        if skip is not None and take is not None:
+        # here we don't want to slice an empty table, or it may segfault
+        if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)):
             pa_table = pa_table.slice(skip, take)
         return pa_table
 
@@ -317,7 +318,8 @@ def _get_dataset_from_filename(self, filename_skip_take):
             filename_skip_take["take"] if "take" in filename_skip_take else None,
         )
         pa_table = pa.parquet.read_table(filename, memory_map=True)
-        if skip is not None and take is not None:
+        # here we don't want to slice an empty table, or it may segfault
+        if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)):
             pa_table = pa_table.slice(skip, take)
         return pa_table
 

src/nlp/arrow_writer.py

@@ -288,6 +288,11 @@ def write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = Non
 
     def finalize(self, close_stream=True):
         self.write_on_file()
+        if self.pa_writer is None:
+            if self._schema is not None:
+                self._build_writer(self._schema)
+            else:
+                raise ValueError("Please pass `features` or at least one example when writing data")
         self.pa_writer.close()
         if close_stream:
             self.stream.close()