Skip to content

Commit 02944c2

Browse files
rhatdanrhatdan
authored
Merge pull request #1773 from containers/de-dup
De-duplicate bash build scripts
2 parents ef113ef + c95769c commit 02944c2

File tree

3 files changed

+81
-97
lines changed

3 files changed

+81
-97
lines changed

‎container-images/scripts/build-vllm.sh

Lines changed: 4 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,5 @@
11
#!/bin/bash
22

3-
available() {
4-
command -v "$1" >/dev/null
5-
}
6-
7-
is_rhel_based() { # doesn't include openEuler
8-
# shellcheck disable=SC1091
9-
source /etc/os-release
10-
[ "$ID" = "rhel" ] || [ "$ID" = "redhat" ] || [ "$ID" == "centos" ]
11-
}
12-
13-
dnf_install_epel() {
14-
local rpm_exclude_list="selinux-policy,container-selinux"
15-
local url="https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm"
16-
dnf reinstall -y "$url" || dnf install -y "$url" --exclude "$rpm_exclude_list"
17-
crb enable # this is in epel-release, can only install epel-release via url
18-
}
19-
20-
add_stream_repo() {
21-
local url="https://mirror.stream.centos.org/9-stream/$1/$uname_m/os/"
22-
dnf config-manager --add-repo "$url"
23-
url="http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-Official"
24-
local file="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-Official"
25-
if [ ! -e $file ]; then
26-
curl --retry 8 --retry-all-errors -o $file "$url"
27-
rpm --import $file
28-
fi
29-
}
30-
31-
rm_non_ubi_repos() {
32-
local dir="/etc/yum.repos.d"
33-
rm -rf $dir/mirror.stream.centos.org_9-stream_* $dir/epel*
34-
}
35-
363
install_deps() {
374
if available dnf; then
385
dnf install -y git wget ca-certificates gcc gcc-c++ libSM libXext \
@@ -103,15 +70,6 @@ pip_install() {
10370
uv pip install -v -r "$1" --extra-index-url "$url"
10471
}
10572

106-
git_clone_specific_commit() {
107-
local repo="${vllm_url##*/}"
108-
git init "$repo"
109-
cd "$repo"
110-
git remote add origin "$vllm_url"
111-
git fetch --depth 1 origin $commit
112-
git reset --hard $commit
113-
}
114-
11573
pip_install_all() {
11674
if [ "$containerfile" = "ramalama" ]; then
11775
pip_install requirements/cpu-build.txt
@@ -140,6 +98,9 @@ set_vllm_env_vars() {
14098
main() {
14199
set -eux -o pipefail
142100

101+
# shellcheck disable=SC1091
102+
source container-images/scripts/lib.sh
103+
143104
local containerfile=$1
144105
if [ "$containerfile" != "ramalama" ] && [ "$containerfile" != "cuda" ]; then
145106
echo "First argument must be 'ramalama' or 'cuda'. Got: '$containerfile'"
@@ -157,7 +118,7 @@ main() {
157118

158119
local vllm_url="https://github.com/vllm-project/vllm"
159120
local commit="6d8d0a24c02bfd84d46b3016b865a44f048ae84b"
160-
git_clone_specific_commit
121+
git_clone_specific_commit "$vllm_url" "$commit"
161122
set_vllm_env_vars
162123
pip_install_all
163124

‎container-images/scripts/build_llama_and_whisper.sh

Lines changed: 30 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,6 @@ Python 3.10"
1515
echo "python3"
1616
}
1717

18-
available() {
19-
command -v "$1" >/dev/null
20-
}
21-
2218
dnf_install_intel_gpu() {
2319
local intel_rpms=("intel-oneapi-mkl-sycl-devel" "intel-oneapi-dnnl-devel"
2420
"intel-oneapi-compiler-dpcpp-cpp" "intel-level-zero"
@@ -75,26 +71,6 @@ dnf_install_s390() {
7571
dnf install -y "openblas-devel"
7672
}
7773

78-
add_stream_repo() {
79-
local url="https://mirror.stream.centos.org/9-stream/$1/$uname_m/os/"
80-
dnf config-manager --add-repo "$url"
81-
url="http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-Official"
82-
local file="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-Official"
83-
if [ ! -e $file ]; then
84-
curl --retry 8 --retry-all-errors -o $file "$url"
85-
rpm --import $file
86-
fi
87-
}
88-
89-
rm_non_ubi_repos() {
90-
local dir="/etc/yum.repos.d"
91-
rm -rf $dir/mirror.stream.centos.org_9-stream_* $dir/epel*
92-
}
93-
94-
is_rhel_based() { # doesn't include openEuler
95-
[[ "${ID}" == "rhel" || "${ID}" == "redhat" || "${ID}" == "centos" ]]
96-
}
97-
9874
dnf_install_mesa() {
9975
if [ "${ID}" = "fedora" ]; then
10076
dnf copr enable -y slp/mesa-libkrun-vulkan
@@ -110,13 +86,6 @@ dnf_install_mesa() {
11086
rm_non_ubi_repos
11187
}
11288

113-
dnf_install_epel() {
114-
local rpm_exclude_list="selinux-policy,container-selinux"
115-
local url="https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm"
116-
dnf reinstall -y "$url" || dnf install -y "$url" --exclude "${rpm_exclude_list}"
117-
crb enable # this is in epel-release, can only install epel-release via url
118-
}
119-
12089
# There is no ffmpeg-free package in the openEuler repository. openEuler can use ffmpeg,
12190
# which also has the same GPL/LGPL license as ffmpeg-free.
12291
dnf_install_ffmpeg() {
@@ -132,6 +101,7 @@ dnf_install_ffmpeg() {
132101
else
133102
dnf install -y ffmpeg-free
134103
fi
104+
135105
rm_non_ubi_repos
136106
}
137107

@@ -258,10 +228,7 @@ clone_and_build_whisper_cpp() {
258228
whisper_flags+=("-DCMAKE_POSITION_INDEPENDENT_CODE=ON")
259229
fi
260230

261-
git clone https://github.com/ggerganov/whisper.cpp
262-
cd whisper.cpp
263-
git submodule update --init --recursive
264-
git reset --hard "$whisper_cpp_sha"
231+
git_clone_specific_commit "https://github.com/ggerganov/whisper.cpp" "$whisper_cpp_sha"
265232
cmake_steps "${whisper_flags[@]}"
266233
mkdir -p "$install_prefix/bin"
267234
cd ..
@@ -272,10 +239,7 @@ clone_and_build_llama_cpp() {
272239
local llama_cpp_sha="3f4fc97f1d745f1d5d3c853949503136d419e6de"
273240
local install_prefix
274241
install_prefix=$(set_install_prefix)
275-
git clone https://github.com/ggml-org/llama.cpp
276-
cd llama.cpp
277-
git submodule update --init --recursive
278-
git reset --hard "$llama_cpp_sha"
242+
git_clone_specific_commit "https://github.com/ggml-org/llama.cpp" "$llama_cpp_sha"
279243
cmake_steps "${common_flags[@]}"
280244
install -m 755 build/bin/rpc-server "$install_prefix"/bin/rpc-server
281245
cd ..
@@ -301,6 +265,27 @@ install_entrypoints() {
301265
fi
302266
}
303267

268+
cleanup() {
269+
clone_and_build_llama_cpp
270+
available dnf && dnf_remove
271+
rm -rf /var/cache/*dnf* /opt/rocm-*/lib/*/library/*gfx9*
272+
ldconfig # needed for libraries
273+
}
274+
275+
add_common_flags() {
276+
common_flags+=("-DLLAMA_CURL=ON" "-DGGML_RPC=ON")
277+
case "$containerfile" in
278+
ramalama)
279+
if [ "$uname_m" = "x86_64" ] || [ "$uname_m" = "aarch64" ]; then
280+
common_flags+=("-DGGML_VULKAN=ON")
281+
elif [ "$uname_m" = "s390x" ]; then
282+
common_flags+=("-DGGML_VXE=ON" "-DGGML_BLAS=ON")
283+
common_flags+=("-DGGML_BLAS_VENDOR=OpenBLAS")
284+
fi
285+
;;
286+
esac
287+
}
288+
304289
main() {
305290
# shellcheck disable=SC1091
306291
source /etc/os-release
@@ -309,6 +294,9 @@ main() {
309294
export PYTHON
310295
PYTHON=$(python_version)
311296

297+
# shellcheck disable=SC1091
298+
source container-images/scripts/lib.sh
299+
312300
local containerfile=${1-""}
313301
local install_prefix
314302
install_prefix=$(set_install_prefix)
@@ -321,27 +309,15 @@ main() {
321309
if [ -n "$containerfile" ]; then
322310
install_ramalama "${install_prefix}"
323311
fi
324-
install_entrypoints
325312

313+
install_entrypoints
326314
setup_build_env
327315
if [ "$uname_m" != "s390x" ] && [ "$containerfile" != "rocm-ubi" ]; then
328316
clone_and_build_whisper_cpp
329317
fi
330-
common_flags+=("-DLLAMA_CURL=ON" "-DGGML_RPC=ON")
331-
case "$containerfile" in
332-
ramalama)
333-
if [ "$uname_m" = "x86_64" ] || [ "$uname_m" = "aarch64" ]; then
334-
common_flags+=("-DGGML_VULKAN=ON")
335-
elif [ "$uname_m" = "s390x" ]; then
336-
common_flags+=("-DGGML_VXE=ON" "-DGGML_BLAS=ON" "-DGGML_BLAS_VENDOR=OpenBLAS")
337-
fi
338-
;;
339-
esac
340318

341-
clone_and_build_llama_cpp
342-
available dnf && dnf_remove
343-
rm -rf /var/cache/*dnf* /opt/rocm-*/lib/*/library/*gfx9*
344-
ldconfig # needed for libraries
319+
add_common_flags
320+
cleanup
345321
}
346322

347323
main "$@"

‎container-images/scripts/lib.sh

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
available() {
4+
command -v "$1" >/dev/null
5+
}
6+
7+
is_rhel_based() { # doesn't include openEuler
8+
# shellcheck disable=SC1091
9+
source /etc/os-release
10+
[ "$ID" = "rhel" ] || [ "$ID" = "redhat" ] || [ "$ID" == "centos" ]
11+
}
12+
13+
dnf_install_epel() {
14+
local rpm_exclude_list="selinux-policy,container-selinux"
15+
local url="https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm"
16+
dnf reinstall -y "$url" || dnf install -y "$url" --exclude "$rpm_exclude_list"
17+
crb enable # this is in epel-release, can only install epel-release via url
18+
}
19+
20+
add_stream_repo() {
21+
local uname_m
22+
uname_m="$(uname -m)"
23+
local url="https://mirror.stream.centos.org/9-stream/$1/$uname_m/os/"
24+
dnf config-manager --add-repo "$url"
25+
url="http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-Official"
26+
local file="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-Official"
27+
if [ ! -e $file ]; then
28+
curl --retry 8 --retry-all-errors -o $file "$url"
29+
rpm --import $file
30+
fi
31+
}
32+
33+
rm_non_ubi_repos() {
34+
local dir="/etc/yum.repos.d"
35+
rm -rf $dir/mirror.stream.centos.org_9-stream_* $dir/epel*
36+
}
37+
38+
git_clone_specific_commit() {
39+
local repo="${1##*/}"
40+
git init "$repo"
41+
cd "$repo" || return 1
42+
git remote add origin "$1"
43+
git fetch --depth 1 origin "$2"
44+
git reset --hard "$2"
45+
git submodule update --init --recursive
46+
}
47+

0 commit comments

Comments
 (0)