[ci] raise floors on CI dependencies (#6375)

Author: jameslamb

.ci/conda-envs/README.md

@@ -0,0 +1,11 @@
+# conda-envs
+
+This directory contains files used to create `conda` environments for development
+and testing of LightGBM.
+
+The `.txt` files here are intended to be used with `conda create --file`.
+
+For details on that, see the `conda` docs:
+
+* `conda create` docs ([link](https://conda.io/projects/conda/en/latest/commands/create.html))
+* "Managing Environments" ([link](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html))

.ci/conda-envs/ci-core-py37.txt

@@ -0,0 +1,56 @@
+# [description]
+#
+#   Similar to ci-core.txt, but specific to Python 3.7.
+#
+#   Unlike ci-core.txt, this includes a Python version and uses
+#   `=` and `<=` pins to make solves faster and prevent against
+#   issues like https://github.com/microsoft/LightGBM/pull/6370.
+#
+# [usage]
+#
+#   conda create \
+#     --name test-env \
+#     --file ./.ci/conda-envs/ci-core-py37.txt
+#
+
+# python
+python=3.7.*
+
+# direct imports
+cffi=1.15.*
+# older versions of Dask are incompatible with pandas>=2.0, but not all conda packages' metadata accurately reflects that
+#
+# ref: https://github.com/microsoft/LightGBM/issues/6030
+dask=2022.2.*
+distributed=2022.2.*
+joblib=1.3.*
+matplotlib-base=3.5.*
+numpy=1.21.*
+pandas=1.3.*
+pyarrow=9.0.*
+# python-graphviz 0.20.2 is not compatible with Python 3.7
+# ref: https://github.com/microsoft/LightGBM/pull/6370
+python-graphviz=0.20.1
+scikit-learn=1.0.*
+scipy=1.7.*
+
+# testing-only dependencies
+cloudpickle=2.2.*
+pluggy=1.0.*
+psutil=5.9.3
+pytest=7.4.*
+
+# other recursive dependencies, just
+# pinned here to help speed up solves
+bokeh=2.4.*
+fsspec=2023.1.*
+msgpack-python=1.0.*
+pluggy=1.0.*
+pytz=2024.1
+setuptools=59.8.*
+snappy=1.1.*
+tomli=2.0.*
+tornado=6.1.*
+wheel=0.42.*
+zict=2.2.*
+zipp=3.15.*

.ci/conda-envs/ci-core.txt

@@ -0,0 +1,40 @@
+# [description]
+#
+#   Core dependencies used across most LightGBM continuous integration (CI) jobs.
+#
+#   'python' constraint is intentionally omitted, so this file can be reused across
+#   Python versions.
+#
+#   These floors are not the oldest versions LightGBM supports... they're here just to make conda
+#   solves faster, and should generally be the latest versions that work for all CI jobs using this.
+#
+# [usage]
+#
+#   conda create \
+#     --name test-env \
+#     --file ./.ci/conda-envs/ci-core.txt \
+#     python=3.10
+#
+
+# direct imports
+cffi>=1.16
+dask>=2023.5.0
+joblib>=1.3.2
+matplotlib-base>=3.7.3
+numpy>=1.24.4
+pandas>2.0
+pyarrow>=6.0
+python-graphviz>=0.20.3
+scikit-learn>=1.3.2
+scipy>=1.1
+
+# testing-only dependencies
+cloudpickle>=3.0.0
+psutil>=5.9.8
+pytest>=8.1.1
+
+# other recursive dependencies, just
+# pinned here to help speed up solves
+pluggy>=1.4.0
+setuptools>=69.2
+wheel>=0.43

.ci/test.sh

@@ -46,6 +46,7 @@ if [[ "$TASK" == "cpp-tests" ]]; then
     exit 0
 fi
 
+# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
 CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]"
 
 if [[ $TASK == "if-else" ]]; then
@@ -78,10 +79,10 @@ if [[ $TASK == "lint" ]]; then
         ${CONDA_PYTHON_REQUIREMENT} \
         'cmakelint>=1.4.2' \
         'cpplint>=1.6.0' \
-        'matplotlib>=3.8.3' \
+        'matplotlib-base>=3.8.3' \
         'mypy>=1.8.0' \
         'pre-commit>=3.6.0' \
-        'pyarrow>=14.0' \
+        'pyarrow>=6.0' \
         'r-lintr>=3.1'
     source activate $CONDA_ENV
     echo "Linting Python code"
@@ -127,28 +128,18 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
     exit 0
 fi
 
-# older versions of Dask are incompatible with pandas>=2.0, but not all conda packages' metadata accurately reflects that
-#
-# ref: https://github.com/microsoft/LightGBM/issues/6030
-CONSTRAINED_DEPENDENCIES="'dask>=2023.5.0' 'distributed>=2023.5.0' 'pandas>=2.0' python-graphviz"
 if [[ $PYTHON_VERSION == "3.7" ]]; then
-    CONSTRAINED_DEPENDENCIES="'dask' 'distributed' 'python-graphviz<0.20.2' 'pandas<2.0'"
+    CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt"
+else
+    CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt"
 fi
 
-# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
-mamba create -q -y -n $CONDA_ENV \
-    ${CONSTRAINED_DEPENDENCIES} \
-    cffi \
-    cloudpickle \
-    joblib \
-    matplotlib \
-    numpy \
-    psutil \
-    pyarrow \
-    pytest \
+mamba create \
+    -y \
+    -n $CONDA_ENV \
+    ${CONDA_REQUIREMENT_FILES} \
     ${CONDA_PYTHON_REQUIREMENT} \
-    scikit-learn \
-    scipy || exit 1
+|| exit 1
 
 source activate $CONDA_ENV
 
@@ -310,10 +301,10 @@ matplotlib.use\(\"Agg\"\)\
 ' plot_example.py  # prevent interactive window mode
     sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
     # requirements for examples
-    mamba install -q -y -n $CONDA_ENV \
-        h5py \
-        ipywidgets \
-        notebook
+    mamba install -y -n $CONDA_ENV \
+        'h5py>=3.10' \
+        'ipywidgets>=8.1.2' \
+        'notebook>=7.1.2'
     for f in *.py **/*.py; do python $f || exit 1; done  # run all examples
     cd $BUILD_DIRECTORY/examples/python-guide/notebooks
     sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
@@ -325,7 +316,7 @@ matplotlib.use\(\"Agg\"\)\
         dask \
         distributed \
         joblib \
-        matplotlib \
+        matplotlib-base \
         psutil \
         pyarrow \
         python-graphviz \

.ci/test_windows.ps1

@@ -46,23 +46,21 @@ conda config --set always_yes yes --set changeps1 no
 # ref:
 # * https://stackoverflow.com/a/62897729/3986677
 # * https://github.com/microsoft/LightGBM/issues/5899
-conda install brotlipy
+conda install "brotlipy>=0.7"
 
 conda update -q -y conda
-conda create -q -y -n $env:CONDA_ENV `
-  cffi `
-  cloudpickle `
-  joblib `
-  matplotlib `
-  numpy `
-  pandas `
-  psutil `
-  pyarrow `
-  pytest `
-  "python=$env:PYTHON_VERSION[build=*cpython]" `
-  python-graphviz `
-  scikit-learn `
-  scipy ; Check-Output $?
+
+if ($env:PYTHON_VERSION -eq "3.7") {
+  $env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core-py37.txt"
+} else {
+  $env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core.txt"
+}
+
+conda create `
+  -y `
+  -n $env:CONDA_ENV `
+  --file $env:CONDA_REQUIREMENT_FILE `
+  "python=$env:PYTHON_VERSION[build=*cpython]" ; Check-Output $?
 
 if ($env:TASK -ne "bdist") {
   conda activate $env:CONDA_ENV
@@ -124,7 +122,7 @@ if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -e
   cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
   @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
   (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"  # prevent interactive window mode
-  conda install -q -y -n $env:CONDA_ENV "h5py>3.0" ipywidgets notebook
+  conda install -y -n $env:CONDA_ENV "h5py>=3.10" "ipywidgets>=8.1.2" "notebook>=7.1.2"
   foreach ($file in @(Get-ChildItem *.py)) {
     @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
     python $file ; Check-Output $?