Skip to content

Commit 8c8d059

Browse files
trxcllnttrxcllnt
committed
Merge branch 'branch-25.08' of github.com:rapidsai/devcontainers into branch-25.10
2 parents 95e6acf + 8ac0a32 commit 8c8d059

File tree

10 files changed

+124
-131
lines changed

10 files changed

+124
-131
lines changed

‎.devcontainer/cuda12.9-conda/devcontainer.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"args": {
66
"CUDA": "12.9",
77
"PYTHON_PACKAGE_MANAGER": "conda",
8-
"BASE": "rapidsai/devcontainers:25.10-cpp-mambaforge-ubuntu22.04"
8+
"BASE": "rapidsai/devcontainers:25.10-cpp-mambaforge"
99
}
1010
},
1111
"runArgs": [

‎.devcontainer/cuda12.9-pip/devcontainer.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"args": {
66
"CUDA": "12.9",
77
"PYTHON_PACKAGE_MANAGER": "pip",
8-
"BASE": "rapidsai/devcontainers:25.10-cpp-cuda12.9-ucx1.18.0-openmpi5.0.7-ubuntu22.04"
8+
"BASE": "rapidsai/devcontainers:25.10-cpp-cuda12.9-ucx1.18.0-openmpi5.0.7"
99
}
1010
},
1111
"runArgs": [

‎.github/workflows/build-all-rapids-repos.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ jobs:
4242
timeout-minutes: 720
4343
# Prevent the sccache server from shutting down
4444
env: |
45+
CONDA_ENV_CREATE_QUIET=1
4546
SCCACHE_IDLE_TIMEOUT=0
4647
build_command: |
4748
SCCACHE_NO_DAEMON=1 sccache --stop-server

‎features/src/rapids-build-utils/devcontainer-feature.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "NVIDIA RAPIDS devcontainer build utilities",
33
"id": "rapids-build-utils",
4-
"version": "25.10.0",
4+
"version": "25.10.1",
55
"description": "A feature to install the RAPIDS devcontainer build utilities",
66
"containerEnv": {
77
"BASH_ENV": "/etc/bash.bash_env"

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh

Lines changed: 83 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -24,62 +24,15 @@
2424
# shellcheck disable=SC1091
2525
. rapids-generate-docstring;
2626

27-
generate_env_yaml() {
28-
(
29-
(rapids-dependency-file-generator "${@:2}" 2>/dev/null || echo "name: none") \
30-
| (grep -v '^#' || [ "$?" == "1" ]) \
31-
| tee "${1}" 1>/dev/null;
32-
) & true
27+
_generate_env_yaml() {
28+
if rapids-dependency-file-generator "$@" 2>/dev/null \
29+
| grep -v '^#' 2>/dev/null \
30+
| tee "$file" 1>/dev/null; then
31+
echo "$file"
32+
fi
3333
}
3434

35-
make_conda_dependencies() {
36-
local -;
37-
set -euo pipefail;
38-
39-
eval "$(_parse_args --skip '-m,--manifest -o,--omit --repo' "$@" <&0)";
40-
41-
eval "$(rapids-list-repos "${OPTS[@]}")";
42-
43-
# shellcheck disable=SC1091
44-
. devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env make-conda-dependencies';
45-
46-
test ${#exclude[@]} -eq 0 && exclude=();
47-
test ${#include[@]} -eq 0 && include=();
48-
test ${#matrix_entry[@]} -eq 0 && matrix_entry=();
49-
50-
local -a _exclude=();
51-
local exc; for exc in "${exclude[@]}"; do
52-
# append '-f' so each file's contents will be treated as a list of patterns for 'grep'
53-
_exclude+=(-f "${exc}");
54-
done
55-
56-
local -a _include=();
57-
local inc; for inc in "${include[@]}"; do
58-
# append '-f' so each file's contents will be treated as a list of patterns for 'grep'
59-
_include+=(-f "${inc}");
60-
done
61-
62-
local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}";
63-
cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")";
64-
65-
local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}";
66-
python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
67-
68-
local -a _matrix_selectors=(
69-
arch="$(uname -m)"
70-
cuda="${cuda_version}"
71-
py="${python_version}"
72-
);
73-
74-
# add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
75-
test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}");
76-
local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}")
77-
78-
local conda_env_yamls=();
79-
80-
local i;
81-
local j;
82-
35+
_generate_env_yamls() {
8336
for ((i=0; i < ${repos_length:-0}; i+=1)); do
8437

8538
local repo="repos_${i}";
@@ -113,15 +66,12 @@ make_conda_dependencies() {
11366
local keyi;
11467

11568
for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
116-
local file="/tmp/${name}.${keys[$keyi]}.env.yaml";
117-
conda_env_yamls+=("${file}");
118-
generate_env_yaml \
119-
"${file}" \
69+
local file="${tmpdir}/${name}.${keys[$keyi]}.env.yaml";
70+
_generate_env_yaml \
12071
--file-key "${keys[$keyi]}" \
12172
--output conda \
12273
--config ~/"${path}/dependencies.yaml" \
123-
--matrix "${matrix_selectors}" \
124-
;
74+
--matrix "${matrix_selectors}" &
12575
done
12676

12777
local cpp_length="${repo}_cpp_length";
@@ -136,15 +86,12 @@ make_conda_dependencies() {
13686
local keyi;
13787

13888
for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
139-
local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.env.yaml";
140-
conda_env_yamls+=("${file}");
141-
generate_env_yaml \
142-
"${file}" \
89+
local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.env.yaml";
90+
_generate_env_yaml \
14391
--file-key "${keys[$keyi]}" \
14492
--output conda \
14593
--config ~/"${path}/dependencies.yaml" \
146-
--matrix "${matrix_selectors}" \
147-
;
94+
--matrix "${matrix_selectors}" &
14895
done
14996
done
15097

@@ -160,30 +107,86 @@ make_conda_dependencies() {
160107
local keyi;
161108

162109
for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
163-
local file="/tmp/${name}.${py_name}.${keys[$keyi]}.env.yaml";
164-
conda_env_yamls+=("${file}");
165-
generate_env_yaml \
166-
"${file}" \
110+
local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.env.yaml";
111+
_generate_env_yaml \
167112
--file-key "${keys[$keyi]}" \
168113
--output conda \
169114
--config ~/"${path}/dependencies.yaml" \
170-
--matrix "${matrix_selectors}" \
171-
;
115+
--matrix "${matrix_selectors}" &
172116
done
173117
done
174118
fi
175119
done
120+
}
121+
122+
_make_conda_dependencies() {
123+
local -;
124+
set -euo pipefail;
125+
126+
eval "$(_parse_args --skip '-m,--manifest -o,--omit --repo' "$@" <&0)";
127+
128+
eval "$(rapids-list-repos "${OPTS[@]}")";
129+
130+
# shellcheck disable=SC1091
131+
. devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env make-conda-dependencies';
132+
133+
test ${#exclude[@]} -eq 0 && exclude=();
134+
test ${#include[@]} -eq 0 && include=();
135+
test ${#matrix_entry[@]} -eq 0 && matrix_entry=();
136+
137+
local -a _exclude=();
138+
local exc; for exc in "${exclude[@]}"; do
139+
# append '-f' so each file's contents will be treated as a list of patterns for 'grep'
140+
_exclude+=(-f "${exc}");
141+
done
142+
143+
local -a _include=();
144+
local inc; for inc in "${include[@]}"; do
145+
# append '-f' so each file's contents will be treated as a list of patterns for 'grep'
146+
_include+=(-f "${inc}");
147+
done
148+
149+
local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}";
150+
cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")";
151+
152+
local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}";
153+
python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
154+
155+
local -a _matrix_selectors=(
156+
arch="$(uname -m)"
157+
cuda="${cuda_version}"
158+
py="${python_version}"
159+
);
160+
161+
# add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
162+
test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}");
163+
local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}")
164+
165+
local conda_env_yamls=();
166+
167+
local i;
168+
local j;
169+
170+
local -r tmpdir="$(mktemp -d)";
171+
# shellcheck disable=SC2064
172+
trap "rm -rf '${tmpdir}'" EXIT;
173+
174+
readarray -t conda_env_yamls < <(_generate_env_yamls);
176175

177176
if test ${#conda_env_yamls[@]} -gt 0; then
178177

179-
for ((i=0; i < ${#conda_env_yamls[@]}; i+=1)); do
180-
while ! test -f "${conda_env_yamls[$i]}"; do
181-
sleep 0.1;
182-
done
183-
done
178+
readarray -t rapids_python_pkg_names < <(rapids-python-pkg-names);
179+
readarray -t rapids_python_conda_pkg_names < <(rapids-python-conda-pkg-names);
184180

185-
# shellcheck disable=SC2207
186-
local conda_noinstall=($(rapids-python-pkg-names) $(rapids-python-conda-pkg-names));
181+
local -a conda_noinstall=();
182+
183+
conda_noinstall+=("${rapids_python_pkg_names[@]}");
184+
conda_noinstall+=("${rapids_python_pkg_names[@]//"-"/"_"}");
185+
conda_noinstall+=("${rapids_python_pkg_names[@]//"_"/"-"}");
186+
187+
conda_noinstall+=("${rapids_python_conda_pkg_names[@]}");
188+
conda_noinstall+=("${rapids_python_conda_pkg_names[@]//"-"/"_"}");
189+
conda_noinstall+=("${rapids_python_conda_pkg_names[@]//"_"/"-"}");
187190

188191
# Generate a combined conda env yaml file.
189192
conda-merge "${conda_env_yamls[@]}" \
@@ -193,9 +196,7 @@ make_conda_dependencies() {
193196
| ( if test ${#_exclude[@]} -gt 0; then grep -E -v "${_exclude[@]}" || [ "$?" == "1" ]; else cat -; fi ) \
194197
| ( if test ${#_include[@]} -gt 0; then grep -E "${_include[@]}" || [ "$?" == "1" ]; else cat -; fi ) \
195198
;
196-
197-
rm -f "${conda_env_yamls[@]}";
198199
fi
199200
}
200201

201-
make_conda_dependencies "$@" <&0;
202+
_make_conda_dependencies "$@" <&0;

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ make_conda_env() {
2525
# shellcheck disable=SC1091
2626
. devcontainer-utils-debug-output 'rapids_build_utils_debug' 'make-conda-env';
2727

28-
test ${#quiet[@]} -eq 0 && quiet=(${CONDA_ENV_CREATE_QUIET:+"-q"});
28+
test ${#q[@]} -eq 0 && q=(${CONDA_ENV_CREATE_QUIET:+"-q"});
2929

3030
local env_name="${1}"; shift;
3131
local env_file_name="${env_name}.yml";

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh

Lines changed: 30 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,13 @@
2626
# shellcheck disable=SC1091
2727
. rapids-generate-docstring;
2828

29-
generate_requirements() {
30-
(
31-
(rapids-dependency-file-generator "${@:2}" 2>/dev/null || echo "") \
32-
| (grep -v '^#' || [ "$?" == "1" ]) \
33-
| tee "${1}" 1>/dev/null;
34-
) & true
29+
_generate_requirements() {
30+
(rapids-dependency-file-generator "$@" 2>/dev/null || echo -n) \
31+
| (grep -v '^#' || [ "$?" == "1" ]) \
32+
| tee -a "${reqs}" 1>/dev/null;
3533
}
3634

37-
make_pip_dependencies() {
35+
_make_pip_dependencies() {
3836
local -;
3937
set -euo pipefail;
4038

@@ -96,6 +94,16 @@ make_pip_dependencies() {
9694
local i;
9795
local j;
9896

97+
local -r tmpdir="$(mktemp -d)";
98+
# shellcheck disable=SC2064
99+
trap "rm -rf '${tmpdir}'" EXIT;
100+
local reqs="${tmpdir}/requirements.txt";
101+
mkfifo "${reqs}";
102+
103+
if test ${#requirement[@]} -gt 0; then
104+
cat "${requirement[@]}" | tee -a "$reqs";
105+
fi
106+
99107
for ((i=0; i < ${repos_length:-0}; i+=1)); do
100108

101109
local repo="repos_${i}";
@@ -131,13 +139,12 @@ make_pip_dependencies() {
131139
for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
132140
local file="/tmp/${name}.${keys[$keyi]}.requirements.txt";
133141
pip_reqs_txts+=("${file}");
134-
generate_requirements \
135-
"${file}" \
142+
_generate_requirements \
136143
--file-key "${keys[$keyi]}" \
137144
--output requirements \
138145
--config ~/"${path}/dependencies.yaml" \
139146
--matrix "${matrix_selectors}" \
140-
;
147+
&
141148
done
142149

143150
local cpp_length="${repo}_cpp_length";
@@ -154,13 +161,12 @@ make_pip_dependencies() {
154161
for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
155162
local file="/tmp/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt";
156163
pip_reqs_txts+=("${file}");
157-
generate_requirements \
158-
"${file}" \
164+
_generate_requirements \
159165
--file-key "${keys[$keyi]}" \
160166
--output requirements \
161167
--config ~/"${path}/dependencies.yaml" \
162168
--matrix "${matrix_selectors}" \
163-
;
169+
&
164170
done
165171
done
166172

@@ -178,41 +184,31 @@ make_pip_dependencies() {
178184
for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
179185
local file="/tmp/${name}.${py_name}.${keys[$keyi]}.requirements.txt";
180186
pip_reqs_txts+=("${file}");
181-
generate_requirements \
182-
"${file}" \
187+
_generate_requirements \
183188
--file-key "${keys[$keyi]}" \
184189
--output requirements \
185190
--config ~/"${path}/dependencies.yaml" \
186191
--matrix "${matrix_selectors}" \
187-
;
192+
&
188193
done
189194
done
190195
fi
191196
done
192197

193198
if test ${#requirement[@]} -gt 0 || test ${#pip_reqs_txts[@]} -gt 0; then
194199

195-
for ((i=0; i < ${#pip_reqs_txts[@]}; i+=1)); do
196-
while ! test -f "${pip_reqs_txts[$i]}"; do
197-
sleep 0.1;
198-
done
199-
done
200-
201-
local pip_noinstall=();
200+
readarray -t rapids_python_pkg_names < <(rapids-python-pkg-names);
202201

203202
# add all python packages to the noinstall list.
204-
for pkg in $(rapids-python-pkg-names); do
205-
pip_noinstall+=("${pkg}" "${pkg}-cu.*");
206-
if test -z "${pkg##*"-"*}"; then
207-
pip_noinstall+=("${pkg//"-"/"_"}" "${pkg//"-"/"_"}-cu.*")
208-
fi
209-
if test -z "${pkg##*"_"*}"; then
210-
pip_noinstall+=("${pkg//"_"/"-"}" "${pkg//"_"/"-"}-cu.*")
211-
fi
212-
done
203+
local -a pip_noinstall=();
204+
pip_noinstall+=("${rapids_python_pkg_names[@]}");
205+
pip_noinstall+=("${rapids_python_pkg_names[@]//"-"/"_"}");
206+
pip_noinstall+=("${rapids_python_pkg_names[@]//"_"/"-"}");
207+
pip_noinstall+=("${pip_noinstall[@]/%/"-cu.*"}");
213208

214209
# Generate a combined requirements.txt file
215-
cat "${requirement[@]}" "${pip_reqs_txts[@]}" \
210+
# shellcheck disable=SC2002
211+
cat "${reqs}" \
216212
| (grep -v '^#' || [ "$?" == "1" ]) \
217213
| (grep -v -E '^$' || [ "$?" == "1" ]) \
218214
| ( if test -n "${no_dedupe:+x}"; then cat -; else tr -s "[:blank:]" | LC_ALL=C sort -u; fi ) \
@@ -223,9 +219,7 @@ make_pip_dependencies() {
223219
| sed -E "s/^cupy-cuda[0-9]+x/cupy-cuda${cuda_version_major}x/g" \
224220
| sed -E "s/^cuda-python.*/cuda-python>=${cuda_version}.0,<$((cuda_version_major+1)).0a0/g" \
225221
;
226-
227-
rm -f "${pip_reqs_txts[@]}";
228222
fi
229223
}
230224

231-
make_pip_dependencies "$@" <&0;
225+
_make_pip_dependencies "$@" <&0;

0 commit comments

Comments
 (0)