Skip to content
This repository was archived by the owner on Jan 15, 2024. It is now read-only.

Commit 1c840b4

Browse files
leezuleezu
committed
Parallelize notebook runs via AWS Batch Jobs
1 parent 36c9f65 commit 1c840b4

File tree

4 files changed

+122
-33
lines changed

4 files changed

+122
-33
lines changed

‎Makefile

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,16 @@ clean:
6060

6161
compile_notebooks:
6262
for f in $(shell find docs/examples -type f -name '*.md' -print) ; do \
63-
DIR=`dirname $$f` ; \
64-
BASENAME=`basename $$f` ; \
65-
echo $$DIR $$BASENAME ; \
63+
DIR=$$(dirname $$f) ; \
64+
BASENAME=$$(basename $$f) ; \
65+
TARGETNAME=$${BASENAME%.md}.ipynb ; \
66+
echo $$DIR $$BASENAME $$TARGETNAME; \
6667
cd $$DIR ; \
67-
python $(MD2IPYNB) $$BASENAME ; \
68+
if [ -f $$TARGETNAME ]; then \
69+
echo $$TARGETNAME exists. Skipping compilation of $$BASENAME in Makefile. ; \
70+
else \
71+
python $(MD2IPYNB) $$BASENAME ; \
72+
fi ; \
6873
cd - ; \
6974
done;
7075

‎ci/jenkins/Jenkinsfile_py3-master_gpu_doc

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,84 @@ utils.main_wrapper(
3535
core_logic: {
3636
utils.parallel_stage('Doc Test', [
3737
build_steps.test_doctest('gluon-nlp-cpu-py3', 'cpu/py3',
38-
'src/gluonnlp', 'src/gluonnlp', 4)
38+
'src/gluonnlp', 'src/gluonnlp', 4)
3939
])
4040

41+
// Compile example notebooks, Doctest & Create Website
42+
node { // Single node parallelism
43+
ws('gluon-nlp-cpu-py3') {
44+
stage("Prepare conda environment for website") {
45+
utils.init_git()
46+
// Require a full environment here due to sphinx build step
47+
// after compiling and downloading the notebooks
48+
sh 'source ci/prepare_clean_env.sh cpu/py3'
49+
}
50+
51+
stage("Create Website") {
52+
def tests = [:]
53+
for (f in findFiles(glob: '**/docs/examples/*/*.md')) {
54+
def md_file = f.toString() // Convert FileWrapper to String
55+
def short_name = md_file["docs/examples/".length()..-1]
56+
tests[short_name] = { ->
57+
def base_name = md_file[0..-4] + ''
58+
def ipynb_file = base_name + '.ipynb'
59+
def stdout_file = base_name + '.stdout.log'
60+
def stderr_file = base_name + '.stderr.log'
61+
stage(short_name) { // remove common path from name
62+
// Submit AWS Batch jobs for each example notebook
63+
// The converted notebooks and the conversion logs are
64+
// saved to S3 and retrieved on the CI server once the jobs
65+
// finished.
66+
sh """
67+
conda activate ./conda/cpu/py3
68+
69+
python3 ci/batch/submit-job.py --region us-east-1 --wait \
70+
--timeout 1800 --saved-output ./docs/examples --conda-env docker/py3 \
71+
--name GluonNLP-${env.BRANCH_NAME}-${env.BUILD_NUMBER} \
72+
--save-path batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/docs/examples \
73+
--work-dir . --source-ref refs/pull/${env.CHANGE_ID}/head \
74+
--command \"python3 docs/md2ipynb.py ${md_file} > ${stdout_file} 2> ${stderr_file} \"
75+
BATCH_EXIT_CODE=\$?
76+
77+
aws s3api wait object-exists --bucket gluon-nlp-staging \
78+
--key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stderr_file}
79+
aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stderr_file} ${stderr_file}
80+
cat ${stderr_file}
81+
82+
aws s3api wait object-exists --bucket gluon-nlp-staging \
83+
--key batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stdout_file}
84+
aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${stdout_file} ${stdout_file}
85+
cat ${stdout_file}
86+
87+
aws s3 cp s3://gluon-nlp-staging/batch/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${ipynb_file} ${ipynb_file}
88+
89+
exit \$BATCH_EXIT_CODE
90+
"""
91+
}
92+
}
93+
}
94+
95+
parallel tests
96+
}
97+
98+
stage("Upload Website") {
99+
if (env.BRANCH_NAME.startsWith('PR-')){
100+
bucket = 'gluon-nlp-staging'
101+
path = env.BRANCH_NAME+'/'+env.BUILD_NUMBER
102+
} else {
103+
bucket = 'gluon-nlp'
104+
path = env.BRANCH_NAME
105+
}
106+
sh """
107+
conda activate ./conda/cpu/py3
108+
make docs
109+
ci/upload_doc.sh ${bucket} ${path}
110+
"""
111+
}
112+
}
113+
}
114+
41115
utils.parallel_stage('Documentation', [
42-
build_steps.create_website('gluon-nlp-gpu-py3', 'gpu/py3'),
43116
build_steps.website_linkcheck('gluon-nlp-cpu-py3', 'cpu/py3')
44117
])
45118

‎ci/jenkins/build_steps.groovy

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ def test_unittest(workspace_name, conda_env_name,
6363
def test_doctest(workspace_name, conda_env_name,
6464
test_path, cov_path, threads) {
6565
capture_flag = env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no'
66-
return ["${conda_env_name}: doctest ${test_path}'": {
66+
return ["${conda_env_name}: doctest ${test_path}": {
6767
node(NODE_LINUX_CPU) {
6868
ws(workspace_name) {
6969
utils.init_git()
@@ -79,34 +79,9 @@ def test_doctest(workspace_name, conda_env_name,
7979
}]
8080
}
8181

82-
def create_website(workspace_name, conda_env_name) {
83-
if (env.BRANCH_NAME.startsWith('PR-')){
84-
bucket = 'gluon-nlp-staging'
85-
path = env.BRANCH_NAME+'/'+env.BUILD_NUMBER
86-
} else {
87-
bucket = 'gluon-nlp'
88-
path = env.BRANCH_NAME
89-
}
90-
return ["${conda_env_name}: website'": {
91-
node(NODE_LINUX_GPU) {
92-
ws(workspace_name) {
93-
utils.init_git()
94-
sh """
95-
set -ex
96-
source ci/prepare_clean_env.sh ${conda_env_name}
97-
make docs
98-
99-
ci/upload_doc.sh ${bucket} ${path}
100-
set +ex
101-
"""
102-
}
103-
}
104-
}]
105-
}
106-
10782
def website_linkcheck(workspace_name, conda_env_name) {
10883
enforce_linkcheck = env.BRANCH_NAME.startsWith('PR-')?'false':'true'
109-
return ["${conda_env_name}: website link check'": {
84+
return ["${conda_env_name}: website link check": {
11085
node(NODE_LINUX_CPU) {
11186
ws(workspace_name) {
11287
utils.init_git()

‎env/docker/py3.env

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
channels:
2+
- conda-forge
3+
dependencies:
4+
- python=3.6
5+
- pip=18.1
6+
- perl
7+
- pylint=1.9.2
8+
- flake8
9+
- sphinx=1.7.7
10+
- spacy
11+
- nltk
12+
- pytest=4.5.0
13+
- flaky=3.5.3
14+
- pytest-cov=2.7.1
15+
- mock<3
16+
- pytest-xdist<2
17+
- recommonmark
18+
- pandoc=1.19.2
19+
- notedown
20+
- numba>=v0.40.0
21+
- sphinx-gallery
22+
- nbsphinx>=0.3.4,<0.4
23+
- nbconvert=5.4.0
24+
- tornado=5.1.1
25+
- ipython
26+
- ipykernel
27+
- regex
28+
- pip:
29+
- pylint-quotes<0.2
30+
- mxnet-cu101mkl>=1.4.1
31+
- sacremoses
32+
- sentencepiece<0.2
33+
- https://github.com/mli/mx-theme/tarball/v0.3.9
34+
- seaborn
35+
- jieba
36+
- sphinx-autorun

0 commit comments

Comments
 (0)