Merge branch 'master' into doc-metrics

Author: thomwolf

.circleci/config.yml

@@ -34,12 +34,10 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
-            - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
             - run: sudo pip install .[quality]
-            - run: black --check --line-length 119 --target-version py36 src tests benchmarks
-            - run: isort --check-only --recursive src tests benchmarks
-            - run: flake8 src tests benchmarks
+            - run: black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics
+            - run: isort --check-only tests src benchmarks datasets metrics
+            - run: flake8 tests src benchmarks datasets metrics
     build_doc:
         working_directory: ~/nlp
         docker:

CONTRIBUTING.md

@@ -28,13 +28,6 @@
    it with `pip uninstall nlp` before reinstalling it in editable
    mode with the `-e` flag.)
 
-   Right now, we need an unreleased version of `isort` to avoid a
-   [bug](https://github.com/timothycrosley/isort/pull/1000):
-
-   ```bash
-   $ pip install -U git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
-   ```
-
 5. Develop the features on your branch. If you want to add a dataset see more in-detail intsructions in the section [*How to add a dataset*](#how-to-add-a-dataset). Alternatively, you can follow the steps to [add a dataset](https://huggingface.co/nlp/add_dataset.html) and [share a dataset](https://huggingface.co/nlp/share_dataset.html) in the documentation.
 
 6. Format your code. Run black and isort so that your newly added files look nice with the following command:

Makefile

@@ -3,15 +3,15 @@
 # Check that source code meets quality standards
 
 quality:
-	black --check --line-length 119 --target-version py36 tests src benchmarks
-	isort --check-only --recursive tests src benchmarks datasets
-	flake8 tests src benchmarks
+	black --check --line-length 119 --target-version py36 tests src benchmarks datasets metrics
+	isort --check-only tests src benchmarks datasets metrics
+	flake8 tests src benchmarks datasets metrics
 
 # Format source code automatically
 
 style:
-	black --line-length 119 --target-version py36 tests src benchmarks
-	isort --recursive tests src datasets benchmarks
+	black --line-length 119 --target-version py36 tests src benchmarks datasets metrics
+	isort tests src benchmarks datasets metrics
 
 # Run tests for the library
 

benchmarks/benchmark_array_xd.py

@@ -2,10 +2,11 @@
 import os
 import tempfile
 
+from utils import generate_examples, get_duration
+
 import nlp
 from nlp.arrow_writer import ArrowWriter
 from nlp.features import Array2D
-from utils import generate_examples, get_duration
 
 
 SHAPE_TEST_1 = (30, 487)

benchmarks/benchmark_indices_mapping.py

@@ -2,9 +2,10 @@
 import os
 import tempfile
 
-import nlp
 from utils import generate_example_dataset, get_duration
 
+import nlp
+
 
 SPEED_TEST_N_EXAMPLES = 500_000
 

benchmarks/benchmark_iterating.py

@@ -2,9 +2,10 @@
 import os
 import tempfile
 
-import nlp
 from utils import generate_example_dataset, get_duration
 
+import nlp
+
 
 SPEED_TEST_N_EXAMPLES = 50_000
 SMALL_TEST = 5_000

benchmarks/benchmark_map_filter.py

@@ -3,9 +3,9 @@
 import tempfile
 
 import transformers
+from utils import generate_example_dataset, get_duration
 
 import nlp
-from utils import generate_example_dataset, get_duration
 
 
 SPEED_TEST_N_EXAMPLES = 500_000

datasets/aeslc/aeslc.py

@@ -69,13 +69,16 @@ def _split_generators(self, dl_manager):
         input_path = os.path.join(dl_path, "AESLC-master", "enron_subject_line")
         return [
             nlp.SplitGenerator(
-                name=nlp.Split.TRAIN, gen_kwargs={"pattern": os.path.join(input_path, "train", "*.subject")},
+                name=nlp.Split.TRAIN,
+                gen_kwargs={"pattern": os.path.join(input_path, "train", "*.subject")},
             ),
             nlp.SplitGenerator(
-                name=nlp.Split.VALIDATION, gen_kwargs={"pattern": os.path.join(input_path, "dev", "*.subject")},
+                name=nlp.Split.VALIDATION,
+                gen_kwargs={"pattern": os.path.join(input_path, "dev", "*.subject")},
             ),
             nlp.SplitGenerator(
-                name=nlp.Split.TEST, gen_kwargs={"pattern": os.path.join(input_path, "test", "*.subject")},
+                name=nlp.Split.TEST,
+                gen_kwargs={"pattern": os.path.join(input_path, "test", "*.subject")},
             ),
         ]
 

datasets/ag_news/ag_news.py

@@ -19,25 +19,24 @@
 from __future__ import absolute_import, division, print_function
 
 import csv
-import os
 
 import nlp
 
 
 _DESCRIPTION = """\
-AG is a collection of more than 1 million news articles. News articles have been 
-gathered from more than 2000 news sources by ComeToMyHead in more than 1 year of 
-activity. ComeToMyHead is an academic news search engine which has been running 
-since July, 2004. The dataset is provided by the academic comunity for research 
-purposes in data mining (clustering, classification, etc), information retrieval 
-(ranking, search, etc), xml, data compression, data streaming, and any other 
-non-commercial activity. For more information, please refer to the link 
+AG is a collection of more than 1 million news articles. News articles have been
+gathered from more than 2000 news sources by ComeToMyHead in more than 1 year of
+activity. ComeToMyHead is an academic news search engine which has been running
+since July, 2004. The dataset is provided by the academic comunity for research
+purposes in data mining (clustering, classification, etc), information retrieval
+(ranking, search, etc), xml, data compression, data streaming, and any other
+non-commercial activity. For more information, please refer to the link
 http://www.di.unipi.it/~gulli/AG_corpus_of_news_articles.html .
 
-The AG's news topic classification dataset is constructed by Xiang Zhang 
-([email protected]) from the dataset above. It is used as a text 
-classification benchmark in the following paper: Xiang Zhang, Junbo Zhao, Yann 
-LeCun. Character-level Convolutional Networks for Text Classification. Advances 
+The AG's news topic classification dataset is constructed by Xiang Zhang
+([email protected]) from the dataset above. It is used as a text
+classification benchmark in the following paper: Xiang Zhang, Junbo Zhao, Yann
+LeCun. Character-level Convolutional Networks for Text Classification. Advances
 in Neural Information Processing Systems 28 (NIPS 2015).
 """
 

datasets/allocine/allocine.py

@@ -32,9 +32,9 @@ class AllocineConfig(nlp.BuilderConfig):
     def __init__(self, **kwargs):
         """BuilderConfig for Allocine.
 
-    Args:
-      **kwargs: keyword arguments forwarded to super.
-    """
+        Args:
+          **kwargs: keyword arguments forwarded to super.
+        """
         super(AllocineConfig, self).__init__(**kwargs)
 
 
@@ -58,7 +58,10 @@ def _info(self):
         return nlp.DatasetInfo(
             description=_DESCRIPTION,
             features=nlp.Features(
-                {"review": nlp.Value("string"), "label": nlp.features.ClassLabel(names=["neg", "pos"]),}
+                {
+                    "review": nlp.Value("string"),
+                    "label": nlp.features.ClassLabel(names=["neg", "pos"]),
+                }
             ),
             supervised_keys=None,
             homepage="https://github.com/TheophileBlard/french-sentiment-analysis-with-bert",