add documentation pipeline

Author: Sheng Zha

ci/jenkins/Jenkinsfile_py3-master_gpu_doc

@@ -0,0 +1,51 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node {
+  // Loading the utilities requires a node context unfortunately
+  checkout scm
+  utils = load('ci/jenkins/utils.groovy')
+  build_steps = load('ci/jenkins/build_steps.groovy')
+}
+utils.assign_node_labels(linux_gpu: 'linux-gpu', linux_cpu: 'linux-cpu')
+
+utils.main_wrapper(
+core_logic: {
+  utils.parallel_stage('Doc Test', [
+    build_steps.test_doctest('gluon-nlp-gpu-py3-master', 'gpu/py3-master',
+                             'src/gluonnlp', 'src/gluonnlp', 4)
+  ])
+
+  utils.parallel_stage('Documentation', [
+    build_steps.create_website('gluon-nlp-gpu-py3-master', 'gpu/py3-master')
+  ])
+
+  utils.parallel_stage('Deploy', [
+    build_steps.post_website_link()
+  ])
+}
+,
+failure_handler: {}
+)

ci/jenkins/build_steps.groovy

@@ -43,22 +43,81 @@ def test_unittest(workspace_name, conda_env_name,
                   mark,
                   threads, gpu, skip_report) {
   capture_flag = env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no'
-  cov_flag = env.BRANCH_NAME.startsWith('PR-')?('PR'+env.CHANGE_ID):env.BRANCH_NAME
   node_type = gpu?NODE_LINUX_GPU:NODE_LINUX_CPU
-  return ["Python2: ${test_path} -m '${mark}'": {
+  return ["${conda_env_name}: ${test_path} -m '${mark}'": {
     node(node_type) {
       ws("workspace/${workspace_name}") {
         utils.init_git()
         sh """
-            set -ex
-            source ci/prepare_clean_env.sh ${conda_env_name}
-            pytest -v ${capture_flag} -n ${threads} -m '${mark}' --durations=30 ${test_path} --cov ${cov_path}
-            set +ex
+        set -ex
+        source ci/prepare_clean_env.sh ${conda_env_name}
+        pytest -v ${capture_flag} -n ${threads} -m '${mark}' --durations=30 --cov ${cov_path} ${test_path}
+        set +ex
         """
         if (!skip_report) utils.publish_test_coverage('GluonNLPCodeCov')
       }
     }
   }]
 }
 
+def test_doctest(workspace_name, conda_env_name,
+                 test_path, cov_path, threads) {
+  capture_flag = env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no'
+  return ["${conda_env_name}: doctest ${test_path}'": {
+    node(NODE_LINUX_CPU) {
+      ws("workspace/${workspace_name}") {
+        utils.init_git()
+        sh """
+        set -ex
+        source ci/prepare_clean_env.sh ${conda_env_name}
+        pytest -v ${capture_flag} -n ${threads} --durations=30 --cov ${cov_path} --doctest-modules ${test_path}
+        set +ex
+        """
+        utils.publish_test_coverage('GluonNLPCodeCov')
+      }
+    }
+  }]
+}
+
+def create_website(workspace_name, conda_env_name) {
+  if (env.BRANCH_NAME.startsWith('PR-')){
+    enforce_linkcheck = 'false'
+    bucket = 'gluon-nlp-staging'
+    path = env.BRANCH_NAME+'/'+env.BUILD_NUMBER
+  } else {
+    enforce_linkcheck = 'true'
+    bucket = 'gluon-nlp'
+    path = env.BRANCH_NAME
+  }
+  return ["${conda_env_name}: website'": {
+    node(NODE_LINUX_CPU) {
+      ws("workspace/${workspace_name}") {
+        utils.init_git()
+        sh """
+        set -ex
+        source ci/prepare_clean_env.sh ${conda_env_name}
+        make docs
+        if [[ ${enforce_linkcheck} == true ]]; then
+            make -C docs linkcheck SPHINXOPTS=-W
+        else
+            set +ex
+            make -C docs linkcheck
+        fi;
+
+        ci/upload_doc.sh ${bucket} ${path}
+        set +ex
+        """
+      }
+    }
+  }]
+}
+
+def post_website_link() {
+  if (env.BRANCH_NAME.startsWith("PR-")) {
+    node {
+      pullRequest.comment("Job ${env.BRANCH_NAME}/${env.BUILD_NUMBER} is complete. \nDocs are uploaded to http://gluon-nlp-staging.s3-accelerate.dualstack.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/index.html")
+    }
+  }
+}
+
 return this

ci/step_doc_test.sh

@@ -31,6 +31,7 @@
 
 import numpy as np
 import mxnet as mx
+import gluonnlp
 import pytest
 
 
@@ -187,3 +188,12 @@ def test_not_ok_with_random_data():
 @pytest.fixture(params=[True, False])
 def hybridize(request):
     return request.param
+
+@pytest.fixture(autouse=True)
+def doctest(doctest_namespace):
+    doctest_namespace['np'] = np
+    doctest_namespace['gluonnlp'] = gluonnlp
+    doctest_namespace['mx'] = mx
+    doctest_namespace['gluon'] = mx.gluon
+    import doctest
+    doctest.ELLIPSIS_MARKER = '-etc-'

conftest.py

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -j 4
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build

docs/Makefile

@@ -61,15 +61,6 @@
     'sphinx_autorun',
 ]
 
-doctest_global_setup = '''
-import gluonnlp
-import mxnet as mx
-from mxnet import gluon
-import numpy as np
-import doctest
-doctest.ELLIPSIS_MARKER = '-etc-'
-'''
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

docs/conf.py

@@ -34,7 +34,7 @@ def test_sanity_gpu_serial():
 
 @pytest.mark.gpu
 @pytest.mark.integration
-def test_sanity_gpu_serial():
+def test_sanity_gpu_integ():
     # sanity test that makes sure every marker combination has at least 1 test.
     # due to https://github.com/pytest-dev/pytest/issues/812
     import gluonnlp as nlp